Module Name: src
Committed By: matt
Date: Sat Sep 1 00:00:42 UTC 2012
Modified Files:
src/sys/arch/arm/pic: files.pic pic.c pic_splfuncs.c picvar.h
Log Message:
Add PIC hooks for MP and for the ARM Generic Interrupt Controller.
To generate a diff of this commit:
cvs rdiff -u -r1.3 -r1.4 src/sys/arch/arm/pic/files.pic \
src/sys/arch/arm/pic/pic_splfuncs.c
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/arm/pic/pic.c
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/arm/pic/picvar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/pic/files.pic
diff -u src/sys/arch/arm/pic/files.pic:1.3 src/sys/arch/arm/pic/files.pic:1.4
--- src/sys/arch/arm/pic/files.pic:1.3 Fri Mar 11 03:16:14 2011
+++ src/sys/arch/arm/pic/files.pic Sat Sep 1 00:00:42 2012
@@ -1,9 +1,11 @@
-# $NetBSD: files.pic,v 1.3 2011/03/11 03:16:14 bsh Exp $
+# $NetBSD: files.pic,v 1.4 2012/09/01 00:00:42 matt Exp $
#
# Configuration info for the common PIC code.
#
define pic
define pic_splfuncs
+defflag opt_arm_intr_impl.h __HAVE_PIC_SET_PRIORITY
+defflag opt_arm_intr_impl.h __HAVE_PIC_SOFTINT
file arch/arm/pic/pic.c pic
file arch/arm/pic/pic_splfuncs.c pic & pic_splfuncs
Index: src/sys/arch/arm/pic/pic_splfuncs.c
diff -u src/sys/arch/arm/pic/pic_splfuncs.c:1.3 src/sys/arch/arm/pic/pic_splfuncs.c:1.4
--- src/sys/arch/arm/pic/pic_splfuncs.c:1.3 Sat Jul 14 07:52:53 2012
+++ src/sys/arch/arm/pic/pic_splfuncs.c Sat Sep 1 00:00:42 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: pic_splfuncs.c,v 1.3 2012/07/14 07:52:53 matt Exp $ */
+/* $NetBSD: pic_splfuncs.c,v 1.4 2012/09/01 00:00:42 matt Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -28,15 +28,15 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.3 2012/07/14 07:52:53 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.4 2012/09/01 00:00:42 matt Exp $");
#define _INTR_PRIVATE
#include <sys/param.h>
-#include <sys/evcnt.h>
-#include <sys/atomic.h>
-#include <sys/malloc.h>
-#include <sys/mallocvar.h>
#include <sys/atomic.h>
+#include <sys/evcnt.h>
+#include <sys/kernel.h>
+
+#include <dev/cons.h>
#include <arm/armreg.h>
#include <arm/cpu.h>
@@ -63,11 +63,13 @@ _spllower(int newipl)
const int oldipl = ci->ci_cpl;
KASSERT(panicstr || newipl <= ci->ci_cpl);
if (newipl < ci->ci_cpl) {
- register_t psw = disable_interrupts(I32_bit);
+ register_t psw = cpsid(I32_bit);
ci->ci_intr_depth++;
pic_do_pending_ints(psw, newipl, NULL);
ci->ci_intr_depth--;
- restore_interrupts(psw);
+ if ((psw & I32_bit) == 0)
+ cpsie(I32_bit);
+ cpu_dosoftints();
}
return oldipl;
}
@@ -77,12 +79,26 @@ splx(int savedipl)
{
struct cpu_info * const ci = curcpu();
KASSERT(savedipl < NIPL);
- if (savedipl < ci->ci_cpl) {
- register_t psw = disable_interrupts(I32_bit);
- ci->ci_intr_depth++;
- pic_do_pending_ints(psw, savedipl, NULL);
- ci->ci_intr_depth--;
- restore_interrupts(psw);
+
+ if (__predict_false(savedipl == ci->ci_cpl)) {
+ return;
}
- ci->ci_cpl = savedipl;
+
+ register_t psw = cpsid(I32_bit);
+ KASSERTMSG(panicstr != NULL || savedipl < ci->ci_cpl,
+ "splx(%d) to a higher ipl than %d", savedipl, ci->ci_cpl);
+
+ ci->ci_intr_depth++;
+ pic_do_pending_ints(psw, savedipl, NULL);
+ ci->ci_intr_depth--;
+ KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
+ ci->ci_cpl, savedipl);
+ cpu_dosoftints();
+ KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
+ ci->ci_cpl, savedipl);
+ if ((psw & I32_bit) == 0)
+ cpsie(I32_bit);
+
+ KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
+ ci->ci_cpl, savedipl);
}
Index: src/sys/arch/arm/pic/pic.c
diff -u src/sys/arch/arm/pic/pic.c:1.12 src/sys/arch/arm/pic/pic.c:1.13
--- src/sys/arch/arm/pic/pic.c:1.12 Fri Jul 20 21:53:57 2012
+++ src/sys/arch/arm/pic/pic.c Sat Sep 1 00:00:42 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $ */
+/* $NetBSD: pic.c,v 1.13 2012/09/01 00:00:42 matt Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -28,15 +28,17 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.13 2012/09/01 00:00:42 matt Exp $");
#define _INTR_PRIVATE
#include <sys/param.h>
-#include <sys/evcnt.h>
-#include <sys/atomic.h>
-#include <sys/kmem.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
+#include <sys/evcnt.h>
+#include <sys/intr.h>
+#include <sys/kernel.h>
+#include <sys/kmem.h>
+#include <sys/xcall.h>
#include <arm/armreg.h>
#include <arm/cpufunc.h>
@@ -74,10 +76,80 @@ EVCNT_ATTACH_STATIC(pic_deferral_ev);
void
pic_set_priority(struct cpu_info *ci, int newipl)
{
- register_t psw = disable_interrupts(I32_bit);
+ register_t psw = cpsid(I32_bit);
+ if (pic_list[0] != NULL)
+ (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl);
ci->ci_cpl = newipl;
- (pic_list[0]->pic_set_priority)(newipl);
- restore_interrupts(psw);
+ if ((psw & I32_bit) == 0)
+ cpsie(I32_bit);
+}
+#endif
+
+#ifdef MULTIPROCESSOR
+int
+pic_ipi_nop(void *arg)
+{
+ /* do nothing */
+ return 1;
+}
+
+int
+pic_ipi_xcall(void *arg)
+{
+ xc_ipi_handler();
+ return 1;
+}
+
+void
+intr_cpu_init(struct cpu_info *ci)
+{
+ for (size_t slot = 0; slot < PIC_MAXPICS; slot++) {
+ struct pic_softc * const pic = pic_list[slot];
+ if (pic != NULL && pic->pic_ops->pic_cpu_init != NULL) {
+ (*pic->pic_ops->pic_cpu_init)(pic, ci);
+ }
+ }
+}
+
+typedef void (*pic_ipi_send_func_t)(struct pic_softc *, u_long);
+
+static struct pic_softc *
+pic_ipi_sender(void)
+{
+ for (size_t slot = 0; slot < PIC_MAXPICS; slot++) {
+ struct pic_softc * const pic = pic_list[slot];
+ if (pic != NULL && pic->pic_ops->pic_ipi_send != NULL) {
+ return pic;
+ }
+ }
+ return NULL;
+}
+
+void
+intr_ipi_send(const kcpuset_t *kcp, u_long ipi)
+{
+ struct pic_softc * const pic = pic_ipi_sender();
+ KASSERT(ipi < NIPI);
+ if (cold && pic == NULL)
+ return;
+ KASSERT(pic != NULL);
+ (*pic->pic_ops->pic_ipi_send)(pic, kcp, ipi);
+}
+#endif /* MULTIPROCESSOR */
+
+#ifdef __HAVE_PIC_FAST_SOFTINTS
+int
+pic_handle_softint(void *arg)
+{
+ void softint_switch(lwp_t *, int);
+ struct cpu_info * const ci = curcpu();
+ const size_t softint = (size_t) arg;
+ int s = splhigh();
+ ci->ci_intr_depth--; // don't count these as interrupts
+ softint_switch(ci->ci_softlwps[softint], s);
+ ci->ci_intr_depth++;
+ splx(s);
+ return 1;
}
#endif
@@ -185,6 +257,7 @@ pic_dispatch(struct intrsource *is, void
{
int rv;
+
if (__predict_false(is->is_arg == NULL)
&& __predict_true(frame != NULL)) {
rv = (*is->is_func)(frame);
@@ -194,7 +267,11 @@ pic_dispatch(struct intrsource *is, void
pic_deferral_ev.ev_count++;
return;
}
- is->is_ev.ev_count++;
+
+ struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu);
+ KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC);
+ pcpu->pcpu_evs[is->is_irq].ev_count++;
+ percpu_putref(is->is_pic->pic_percpu);
}
void
@@ -367,8 +444,10 @@ void
pic_do_pending_ints(register_t psw, int newipl, void *frame)
{
struct cpu_info * const ci = curcpu();
- if (__predict_false(newipl == IPL_HIGH))
+ if (__predict_false(newipl == IPL_HIGH)) {
+ KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl);
return;
+ }
while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
KASSERT(pic_pending_ipls < __BIT(NIPL));
for (;;) {
@@ -384,8 +463,34 @@ pic_do_pending_ints(register_t psw, int
}
if (ci->ci_cpl != newipl)
pic_set_priority(ci, newipl);
-#ifdef __HAVE_FAST_SOFTINTS
- cpu_dosoftints();
+}
+
+static void
+pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci)
+{
+ struct pic_percpu * const pcpu = v0;
+ struct pic_softc * const pic = v1;
+
+ pcpu->pcpu_evs = kmem_zalloc(pic->pic_maxsources * sizeof(pcpu->pcpu_evs[0]),
+ KM_SLEEP);
+ KASSERT(pcpu->pcpu_evs != NULL);
+
+#define PCPU_NAMELEN 32
+ const size_t namelen = strlen(pic->pic_name) + 4 + strlen(ci->ci_data.cpu_name);
+
+ KASSERT(namelen < PCPU_NAMELEN);
+ pcpu->pcpu_name = kmem_alloc(PCPU_NAMELEN, KM_SLEEP);
+#ifdef MULTIPROCESSOR
+ snprintf(pcpu->pcpu_name, PCPU_NAMELEN,
+ "%s (%s)", pic->pic_name, ci->ci_data.cpu_name);
+#else
+ strlcpy(pcpu->pcpu_name, pic->pic_name, PCPU_NAMELEN);
+#endif
+ pcpu->pcpu_magic = PICPERCPU_MAGIC;
+#if 0
+ printf("%s: %s %s: <%s>\n",
+ __func__, ci->ci_data.cpu_name, pic->pic_name,
+ pcpu->pcpu_name);
#endif
}
@@ -394,6 +499,8 @@ pic_add(struct pic_softc *pic, int irqba
{
int slot, maybe_slot = -1;
+ KASSERT(strlen(pic->pic_name) > 0);
+
for (slot = 0; slot < PIC_MAXPICS; slot++) {
struct pic_softc * const xpic = pic_list[slot];
if (xpic == NULL) {
@@ -422,10 +529,32 @@ pic_add(struct pic_softc *pic, int irqba
KASSERT(pic->pic_maxsources <= PIC_MAXSOURCES);
KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES);
+ /*
+ * Allocate a pointer to each cpu's evcnts and then, for each cpu,
+ * allocate its evcnts and then attach an evcnt for each pin.
+ * We can't allocate the evcnt structures directly since
+ * percpu will move the contents of percpu memory around and
+ * corrupt the pointers in the evcnts themselves. Remember, any
+ * problem can be solved with sufficient indirection.
+ */
+ pic->pic_percpu = percpu_alloc(sizeof(struct pic_percpu));
+ KASSERT(pic->pic_percpu != NULL);
+
+ /*
+ * Now allocate the per-cpu evcnts.
+ */
+ percpu_foreach(pic->pic_percpu, pic_percpu_allocate, pic);
+
pic->pic_sources = &pic_sources[pic_sourcebase];
pic->pic_irqbase = irqbase;
pic_sourcebase += pic->pic_maxsources;
pic->pic_id = slot;
+#ifdef __HAVE_PIC_SET_PRIORITY
+ KASSERT((slot == 0) == (pic->pic_ops->pic_set_priority != NULL));
+#endif
+#ifdef MULTIPROCESSOR
+ KASSERT((slot == 0) == (pic->pic_ops->pic_ipi_send != NULL));
+#endif
pic_list[slot] = pic;
}
@@ -442,6 +571,17 @@ pic_alloc_irq(struct pic_softc *pic)
return -1;
}
+static void
+pic_percpu_evcnt_attach(void *v0, void *v1, struct cpu_info *ci)
+{
+ struct pic_percpu * const pcpu = v0;
+ struct intrsource * const is = v1;
+
+ KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC);
+ evcnt_attach_dynamic(&pcpu->pcpu_evs[is->is_irq], EVCNT_TYPE_INTR, NULL,
+ pcpu->pcpu_name, is->is_source);
+}
+
void *
pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
int (*func)(void *), void *arg)
@@ -465,15 +605,17 @@ pic_establish_intr(struct pic_softc *pic
is->is_type = type;
is->is_func = func;
is->is_arg = arg;
-
+
if (pic->pic_ops->pic_source_name)
(*pic->pic_ops->pic_source_name)(pic, irq, is->is_source,
sizeof(is->is_source));
else
snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq);
- evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL,
- pic->pic_name, is->is_source);
+ /*
+ * Now attach the per-cpu evcnts.
+ */
+ percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_attach, is);
pic->pic_sources[irq] = is;
@@ -522,16 +664,31 @@ pic_establish_intr(struct pic_softc *pic
return is;
}
+static void
+pic_percpu_evcnt_deattach(void *v0, void *v1, struct cpu_info *ci)
+{
+ struct pic_percpu * const pcpu = v0;
+ struct intrsource * const is = v1;
+
+ KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC);
+ evcnt_detach(&pcpu->pcpu_evs[is->is_irq]);
+}
+
void
pic_disestablish_source(struct intrsource *is)
{
struct pic_softc * const pic = is->is_pic;
const int irq = is->is_irq;
+ KASSERT(is == pic->pic_sources[irq]);
+
(*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq));
pic->pic_sources[irq] = NULL;
pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL;
- evcnt_detach(&is->is_ev);
+ /*
+ * Now detach the per-cpu evcnts.
+ */
+ percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_deattach, is);
kmem_free(is, sizeof(*is));
}
@@ -539,12 +696,10 @@ pic_disestablish_source(struct intrsourc
void *
intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg)
{
- int slot;
-
KASSERT(!cpu_intr_p());
KASSERT(!cpu_softintr_p());
- for (slot = 0; slot < PIC_MAXPICS; slot++) {
+ for (size_t slot = 0; slot < PIC_MAXPICS; slot++) {
struct pic_softc * const pic = pic_list[slot];
if (pic == NULL || pic->pic_irqbase < 0)
continue;
@@ -562,5 +717,9 @@ void
intr_disestablish(void *ih)
{
struct intrsource * const is = ih;
+
+ KASSERT(!cpu_intr_p());
+ KASSERT(!cpu_softintr_p());
+
pic_disestablish_source(is);
}
Index: src/sys/arch/arm/pic/picvar.h
diff -u src/sys/arch/arm/pic/picvar.h:1.6 src/sys/arch/arm/pic/picvar.h:1.7
--- src/sys/arch/arm/pic/picvar.h:1.6 Sat Jul 14 07:52:53 2012
+++ src/sys/arch/arm/pic/picvar.h Sat Sep 1 00:00:42 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: picvar.h,v 1.6 2012/07/14 07:52:53 matt Exp $ */
+/* $NetBSD: picvar.h,v 1.7 2012/09/01 00:00:42 matt Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -30,6 +30,10 @@
#ifndef _ARM_PIC_PICVAR_H_
#define _ARM_PIC_PICVAR_H_
+#ifdef MULTIPROCESSOR
+#include <sys/kcpuset.h>
+#endif
+
int _splraise(int);
int _spllower(int);
void splx(int);
@@ -38,6 +42,17 @@ const char *
struct pic_softc;
struct intrsource;
+struct cpu_info;
+
+#define IPI_AST 0 /* just get an interrupt */
+#define IPI_XCALL 1 /* xcall */
+#define IPI_NOP 2 /* just get an interrupt (armv6) */
+#ifndef __HAVE_PREEMPTION
+#define NIPI 3
+#else
+#define IPI_KPREEMPT 4 /* cause a preemption */
+#define NIPI 4
+#endif
int pic_handle_intr(void *);
void pic_mark_pending(struct pic_softc *pic, int irq);
@@ -54,10 +69,17 @@ void pic_dispatch(struct intrsource *is,
void *intr_establish(int irq, int ipl, int type, int (*func)(void *),
void *arg);
void intr_disestablish(void *);
+#ifdef MULTIPROCESSOR
+void intr_cpu_init(struct cpu_info *);
+void intr_ipi_send(const kcpuset_t *, u_long ipi);
+#endif
#ifdef _INTR_PRIVATE
+#include "opt_arm_intr_impl.h"
+
#include <sys/evcnt.h>
+#include <sys/percpu.h>
#ifndef PIC_MAXPICS
#define PIC_MAXPICS 32
@@ -77,10 +99,17 @@ struct intrsource {
uint8_t is_ipl; /* IPL_xxx */
uint8_t is_irq; /* local to pic */
uint8_t is_iplidx;
- struct evcnt is_ev;
char is_source[16];
};
+struct pic_percpu {
+ struct evcnt *pcpu_evs;
+ char *pcpu_name;
+ uint32_t pcpu_magic;
+};
+
+#define PICPERCPU_MAGIC 0xfeedface
+
struct pic_softc {
const struct pic_ops *pic_ops;
struct intrsource **pic_sources;
@@ -88,6 +117,7 @@ struct pic_softc {
volatile uint32_t pic_blocked_irqs[(PIC_MAXSOURCES + 31) / 32];
volatile uint32_t pic_pending_ipls;
size_t pic_maxsources;
+ percpu_t *pic_percpu;
uint8_t pic_id;
int16_t pic_irqbase;
char pic_name[14];
@@ -104,6 +134,10 @@ struct pic_ops {
#ifdef __HAVE_PIC_SET_PRIORITY
void (*pic_set_priority)(struct pic_softc *, int);
#endif
+#ifdef MULTIPROCESSOR
+ void (*pic_cpu_init)(struct pic_softc *, struct cpu_info *);
+ void (*pic_ipi_send)(struct pic_softc *, const kcpuset_t *, u_long);
+#endif
};
#ifdef __HAVE_PIC_SET_PRIORITY
@@ -120,6 +154,13 @@ void pic_set_priority(struct cpu_info *,
void pic_add(struct pic_softc *, int);
void pic_do_pending_int(void);
+#ifdef MULTIPROCESSOR
+int pic_ipi_nop(void *);
+int pic_ipi_xcall(void *);
+#endif
+#ifdef __HAVE_PIC_FAST_SOFTINTS
+int pic_handle_softint(void *);
+#endif
extern struct pic_softc * pic_list[PIC_MAXPICS];
#endif /* _INTR_PRIVATE */
