Module Name: src
Committed By: jmcneill
Date: Sat Feb 20 19:30:46 UTC 2021
Modified Files:
src/sys/arch/arm/pic: pic.c pic_splfuncs.c picvar.h
Log Message:
Inline "pic_do_pending_ints" in splx and check ci_pending_ipls to optimize
the common case (hw priority, no cascaded interrupts pending).
This also removes the need for the "pic_pending_used" flag, and should fix
booting on Raspberry Pi 3.
To generate a diff of this commit:
cvs rdiff -u -r1.66 -r1.67 src/sys/arch/arm/pic/pic.c
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/arm/pic/pic_splfuncs.c
cvs rdiff -u -r1.29 -r1.30 src/sys/arch/arm/pic/picvar.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/pic/pic.c
diff -u src/sys/arch/arm/pic/pic.c:1.66 src/sys/arch/arm/pic/pic.c:1.67
--- src/sys/arch/arm/pic/pic.c:1.66 Sat Feb 20 14:51:07 2021
+++ src/sys/arch/arm/pic/pic.c Sat Feb 20 19:30:46 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pic.c,v 1.66 2021/02/20 14:51:07 jmcneill Exp $ */
+/* $NetBSD: pic.c,v 1.67 2021/02/20 19:30:46 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -33,7 +33,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.66 2021/02/20 14:51:07 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.67 2021/02/20 19:30:46 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -60,8 +60,6 @@ __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.66
#if defined(__HAVE_PIC_PENDING_INTRS)
-bool pic_pending_used __read_mostly = false;
-
/*
* This implementation of pending interrupts on a MULTIPROCESSOR system makes
* the assumption that a PIC (pic_softc) shall only have all its interrupts
@@ -74,8 +72,6 @@ static struct pic_softc *
pic_list_find_pic_by_pending_ipl(struct cpu_info *, uint32_t);
static void
pic_deliver_irqs(struct cpu_info *, struct pic_softc *, int, void *);
-static void
- pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *);
#endif /* __HAVE_PIC_PENDING_INTRS */
@@ -261,9 +257,6 @@ pic_mark_pending_source(struct pic_softc
const uint32_t ipl_mask = __BIT(is->is_ipl);
struct cpu_info * const ci = curcpu();
- if (!pic_pending_used)
- pic_pending_used = true;
-
atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5],
__BIT(is->is_irq & 0x1f));
@@ -296,9 +289,6 @@ pic_mark_pending_sources(struct pic_soft
if (pending == 0)
return ipl_mask;
- if (!pic_pending_used)
- pic_pending_used = true;
-
KASSERT((irq_base & 31) == 0);
(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
@@ -480,7 +470,7 @@ pic_deliver_irqs(struct cpu_info *ci, st
atomic_and_32(&ci->ci_pending_pics, ~__BIT(pic->pic_id));
}
-static void
+void
pic_list_unblock_irqs(struct cpu_info *ci)
{
uint32_t blocked_pics = ci->ci_blocked_pics;
Index: src/sys/arch/arm/pic/pic_splfuncs.c
diff -u src/sys/arch/arm/pic/pic_splfuncs.c:1.13 src/sys/arch/arm/pic/pic_splfuncs.c:1.14
--- src/sys/arch/arm/pic/pic_splfuncs.c:1.13 Tue Feb 16 22:12:50 2021
+++ src/sys/arch/arm/pic/pic_splfuncs.c Sat Feb 20 19:30:46 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pic_splfuncs.c,v 1.13 2021/02/16 22:12:50 jmcneill Exp $ */
+/* $NetBSD: pic_splfuncs.c,v 1.14 2021/02/20 19:30:46 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -28,7 +28,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.13 2021/02/16 22:12:50 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.14 2021/02/20 19:30:46 jmcneill Exp $");
#define _INTR_PRIVATE
#include <sys/param.h>
@@ -46,12 +46,6 @@ __KERNEL_RCSID(0, "$NetBSD: pic_splfuncs
#include <arm/pic/picvar.h>
-#if defined(__HAVE_PIC_PENDING_INTRS)
-extern bool pic_pending_used;
-#else
-#define pic_pending_used false
-#endif
-
int
_splraise(int newipl)
{
@@ -70,16 +64,12 @@ _spllower(int newipl)
const int oldipl = ci->ci_cpl;
KASSERT(panicstr || newipl <= ci->ci_cpl);
if (newipl < ci->ci_cpl) {
- if (__predict_false(pic_pending_used)) {
- register_t psw = cpsid(I32_bit);
- ci->ci_intr_depth++;
- pic_do_pending_ints(psw, newipl, NULL);
- ci->ci_intr_depth--;
- if ((psw & I32_bit) == 0 || newipl == IPL_NONE)
- cpsie(I32_bit);
- } else {
- pic_set_priority(ci, newipl);
- }
+ register_t psw = cpsid(I32_bit);
+ ci->ci_intr_depth++;
+ pic_do_pending_ints(psw, newipl, NULL);
+ ci->ci_intr_depth--;
+ if ((psw & I32_bit) == 0 || newipl == IPL_NONE)
+ cpsie(I32_bit);
cpu_dosoftints();
}
return oldipl;
@@ -95,27 +85,32 @@ splx(int savedipl)
return;
}
- if (__predict_false(pic_pending_used)) {
- register_t psw = cpsid(I32_bit);
- KASSERTMSG(panicstr != NULL || savedipl < ci->ci_cpl,
- "splx(%d) to a higher ipl than %d", savedipl, ci->ci_cpl);
+ register_t psw = cpsid(I32_bit);
- if ((psw & I32_bit) == 0) {
- ci->ci_intr_depth++;
- pic_do_pending_ints(psw, savedipl, NULL);
- ci->ci_intr_depth--;
- KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
- ci->ci_cpl, savedipl);
+#if defined(__HAVE_PIC_PENDING_INTRS)
+ if (__predict_true(ci->ci_pending_ipls == 0)) {
+ goto skip_pending;
+ }
- cpsie(I32_bit);
- cpu_dosoftints();
- } else {
- pic_set_priority_psw(ci, psw, savedipl);
+ while ((ci->ci_pending_ipls & ~__BIT(savedipl)) > __BIT(savedipl)) {
+ KASSERT(ci->ci_pending_ipls < __BIT(NIPL));
+ for (;;) {
+ int ipl = 31 - __builtin_clz(ci->ci_pending_ipls);
+ KASSERT(ipl < NIPL);
+ if (ipl <= savedipl) {
+ break;
+ }
+
+ pic_set_priority_psw(ci, psw, ipl);
+ pic_list_deliver_irqs(ci, psw, ipl, NULL);
+ pic_list_unblock_irqs(ci);
}
- } else {
- pic_set_priority(ci, savedipl);
- cpu_dosoftints();
}
+skip_pending:
+#endif
+
+ pic_set_priority_psw(ci, psw, savedipl);
+ cpu_dosoftints();
KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d",
ci->ci_cpl, savedipl);
Index: src/sys/arch/arm/pic/picvar.h
diff -u src/sys/arch/arm/pic/picvar.h:1.29 src/sys/arch/arm/pic/picvar.h:1.30
--- src/sys/arch/arm/pic/picvar.h:1.29 Sat Feb 20 18:18:53 2021
+++ src/sys/arch/arm/pic/picvar.h Sat Feb 20 19:30:46 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: picvar.h,v 1.29 2021/02/20 18:18:53 jmcneill Exp $ */
+/* $NetBSD: picvar.h,v 1.30 2021/02/20 19:30:46 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -73,6 +73,8 @@ void pic_mark_pending(struct pic_softc *
void pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is);
uint32_t pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
uint32_t pending);
+void pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *);
+void pic_list_unblock_irqs(struct cpu_info *);
#endif /* __HAVE_PIC_PENDING_INTRS */
void *pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
int (*func)(void *), void *arg, const char *);
