Module Name: src
Committed By: jmcneill
Date: Sat Feb 20 14:51:07 UTC 2021
Modified Files:
src/sys/arch/aarch64/include: cpu.h
src/sys/arch/arm/include: cpu.h
src/sys/arch/arm/pic: pic.c
Log Message:
Move 'struct pic_pending' from percpu to struct cpu_info. Saves a few
instructions in splx.
To generate a diff of this commit:
cvs rdiff -u -r1.30 -r1.31 src/sys/arch/aarch64/include/cpu.h
cvs rdiff -u -r1.114 -r1.115 src/sys/arch/arm/include/cpu.h
cvs rdiff -u -r1.65 -r1.66 src/sys/arch/arm/pic/pic.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/include/cpu.h
diff -u src/sys/arch/aarch64/include/cpu.h:1.30 src/sys/arch/aarch64/include/cpu.h:1.31
--- src/sys/arch/aarch64/include/cpu.h:1.30 Mon Dec 7 10:56:12 2020
+++ src/sys/arch/aarch64/include/cpu.h Sat Feb 20 14:51:07 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.30 2020/12/07 10:56:12 jmcneill Exp $ */
+/* $NetBSD: cpu.h,v 1.31 2021/02/20 14:51:07 jmcneill Exp $ */
/*-
* Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -103,6 +103,9 @@ struct cpu_info {
int ci_cpl;
volatile u_int ci_softints;
volatile u_int ci_intr_depth;
+ volatile uint32_t ci_blocked_pics;
+ volatile uint32_t ci_pending_pics;
+ volatile uint32_t ci_pending_ipls;
int ci_kfpu_spl;
Index: src/sys/arch/arm/include/cpu.h
diff -u src/sys/arch/arm/include/cpu.h:1.114 src/sys/arch/arm/include/cpu.h:1.115
--- src/sys/arch/arm/include/cpu.h:1.114 Mon Aug 17 01:52:59 2020
+++ src/sys/arch/arm/include/cpu.h Sat Feb 20 14:51:06 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.114 2020/08/17 01:52:59 mrg Exp $ */
+/* $NetBSD: cpu.h,v 1.115 2021/02/20 14:51:06 jmcneill Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -189,6 +189,9 @@ struct cpu_info {
volatile u_int ci_intr_depth; /* */
volatile u_int ci_softints;
+ volatile uint32_t ci_blocked_pics;
+ volatile uint32_t ci_pending_pics;
+ volatile uint32_t ci_pending_ipls;
lwp_t * ci_lastlwp; /* last lwp */
Index: src/sys/arch/arm/pic/pic.c
diff -u src/sys/arch/arm/pic/pic.c:1.65 src/sys/arch/arm/pic/pic.c:1.66
--- src/sys/arch/arm/pic/pic.c:1.65 Tue Feb 16 22:12:49 2021
+++ src/sys/arch/arm/pic/pic.c Sat Feb 20 14:51:07 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pic.c,v 1.65 2021/02/16 22:12:49 jmcneill Exp $ */
+/* $NetBSD: pic.c,v 1.66 2021/02/20 14:51:07 jmcneill Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -33,7 +33,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.65 2021/02/16 22:12:49 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.66 2021/02/20 14:51:07 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -68,37 +68,15 @@ bool pic_pending_used __read_mostly = fa
* come from the same CPU. In other words, interrupts from a single PIC will
* not be distributed among multiple CPUs.
*/
-struct pic_pending {
- volatile uint32_t blocked_pics;
- volatile uint32_t pending_pics;
- volatile uint32_t pending_ipls;
-};
static uint32_t
pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
static struct pic_softc *
- pic_list_find_pic_by_pending_ipl(struct pic_pending *, uint32_t);
+ pic_list_find_pic_by_pending_ipl(struct cpu_info *, uint32_t);
static void
- pic_deliver_irqs(struct pic_pending *, struct pic_softc *, int, void *);
+ pic_deliver_irqs(struct cpu_info *, struct pic_softc *, int, void *);
static void
- pic_list_deliver_irqs(struct pic_pending *, register_t, int, void *);
+ pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *);
-#ifdef MULTIPROCESSOR
-percpu_t *pic_pending_percpu;
-static struct pic_pending *
-pic_pending_get(void)
-{
- return percpu_getref(pic_pending_percpu);
-}
-static void
-pic_pending_put(struct pic_pending *pend)
-{
- percpu_putref(pic_pending_percpu);
-}
-#else
-struct pic_pending pic_pending;
-#define pic_pending_get() (&pic_pending)
-#define pic_pending_put(pend) __nothing
-#endif /* MULTIPROCESSOR */
#endif /* __HAVE_PIC_PENDING_INTRS */
struct pic_softc *pic_list[PIC_MAXPICS];
@@ -281,6 +259,7 @@ void
pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
{
const uint32_t ipl_mask = __BIT(is->is_ipl);
+ struct cpu_info * const ci = curcpu();
if (!pic_pending_used)
pic_pending_used = true;
@@ -289,10 +268,8 @@ pic_mark_pending_source(struct pic_softc
__BIT(is->is_irq & 0x1f));
atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
- struct pic_pending *pend = pic_pending_get();
- atomic_or_32(&pend->pending_ipls, ipl_mask);
- atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
- pic_pending_put(pend);
+ atomic_or_32(&ci->ci_pending_ipls, ipl_mask);
+ atomic_or_32(&ci->ci_pending_pics, __BIT(pic->pic_id));
}
void
@@ -313,14 +290,15 @@ pic_mark_pending_sources(struct pic_soft
struct intrsource ** const isbase = &pic->pic_sources[irq_base];
struct intrsource *is;
volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
+ struct cpu_info * const ci = curcpu();
uint32_t ipl_mask = 0;
- if (!pic_pending_used)
- pic_pending_used = true;
-
if (pending == 0)
return ipl_mask;
+ if (!pic_pending_used)
+ pic_pending_used = true;
+
KASSERT((irq_base & 31) == 0);
(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
@@ -338,10 +316,8 @@ pic_mark_pending_sources(struct pic_soft
}
atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
- struct pic_pending *pend = pic_pending_get();
- atomic_or_32(&pend->pending_ipls, ipl_mask);
- atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
- pic_pending_put(pend);
+ atomic_or_32(&ci->ci_pending_ipls, ipl_mask);
+ atomic_or_32(&ci->ci_pending_pics, __BIT(pic->pic_id));
return ipl_mask;
}
@@ -410,7 +386,7 @@ pic_dispatch(struct intrsource *is, void
#if defined(__HAVE_PIC_PENDING_INTRS)
void
-pic_deliver_irqs(struct pic_pending *pend, struct pic_softc *pic, int ipl,
+pic_deliver_irqs(struct cpu_info *ci, struct pic_softc *pic, int ipl,
void *frame)
{
const uint32_t ipl_mask = __BIT(ipl);
@@ -491,7 +467,7 @@ pic_deliver_irqs(struct pic_pending *pen
} while (pending_irqs);
if (blocked_irqs) {
atomic_or_32(iblocked, blocked_irqs);
- atomic_or_32(&pend->blocked_pics, __BIT(pic->pic_id));
+ atomic_or_32(&ci->ci_blocked_pics, __BIT(pic->pic_id));
}
}
@@ -501,15 +477,15 @@ pic_deliver_irqs(struct pic_pending *pen
* about these.
*/
if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0)
- atomic_and_32(&pend->pending_pics, ~__BIT(pic->pic_id));
+ atomic_and_32(&ci->ci_pending_pics, ~__BIT(pic->pic_id));
}
static void
-pic_list_unblock_irqs(struct pic_pending *pend)
+pic_list_unblock_irqs(struct cpu_info *ci)
{
- uint32_t blocked_pics = pend->blocked_pics;
+ uint32_t blocked_pics = ci->ci_blocked_pics;
- pend->blocked_pics = 0;
+ ci->ci_blocked_pics = 0;
for (;;) {
struct pic_softc *pic;
@@ -546,9 +522,9 @@ pic_list_unblock_irqs(struct pic_pending
}
struct pic_softc *
-pic_list_find_pic_by_pending_ipl(struct pic_pending *pend, uint32_t ipl_mask)
+pic_list_find_pic_by_pending_ipl(struct cpu_info *ci, uint32_t ipl_mask)
{
- uint32_t pending_pics = pend->pending_pics;
+ uint32_t pending_pics = ci->ci_pending_pics;
struct pic_softc *pic;
for (;;) {
@@ -565,17 +541,17 @@ pic_list_find_pic_by_pending_ipl(struct
}
void
-pic_list_deliver_irqs(struct pic_pending *pend, register_t psw, int ipl,
+pic_list_deliver_irqs(struct cpu_info *ci, register_t psw, int ipl,
void *frame)
{
const uint32_t ipl_mask = __BIT(ipl);
struct pic_softc *pic;
- while ((pic = pic_list_find_pic_by_pending_ipl(pend, ipl_mask)) != NULL) {
- pic_deliver_irqs(pend, pic, ipl, frame);
+ while ((pic = pic_list_find_pic_by_pending_ipl(ci, ipl_mask)) != NULL) {
+ pic_deliver_irqs(ci, pic, ipl, frame);
KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
}
- atomic_and_32(&pend->pending_ipls, ~ipl_mask);
+ atomic_and_32(&ci->ci_pending_ipls, ~ipl_mask);
}
#endif /* __HAVE_PIC_PENDING_INTRS */
@@ -588,21 +564,19 @@ pic_do_pending_ints(register_t psw, int
return;
}
#if defined(__HAVE_PIC_PENDING_INTRS)
- struct pic_pending *pend = pic_pending_get();
- while ((pend->pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
- KASSERT(pend->pending_ipls < __BIT(NIPL));
+ while ((ci->ci_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
+ KASSERT(ci->ci_pending_ipls < __BIT(NIPL));
for (;;) {
- int ipl = 31 - __builtin_clz(pend->pending_ipls);
+ int ipl = 31 - __builtin_clz(ci->ci_pending_ipls);
KASSERT(ipl < NIPL);
if (ipl <= newipl)
break;
pic_set_priority_psw(ci, psw, ipl);
- pic_list_deliver_irqs(pend, psw, ipl, frame);
- pic_list_unblock_irqs(pend);
+ pic_list_deliver_irqs(ci, psw, ipl, frame);
+ pic_list_unblock_irqs(ci);
}
}
- pic_pending_put(pend);
#endif /* __HAVE_PIC_PENDING_INTRS */
#ifdef __HAVE_PREEMPTION
if (newipl == IPL_NONE && (ci->ci_astpending & __BIT(1))) {
@@ -665,11 +639,6 @@ pic_add(struct pic_softc *pic, int irqba
KASSERT(strlen(pic->pic_name) > 0);
-#if defined(__HAVE_PIC_PENDING_INTRS) && defined(MULTIPROCESSOR)
- if (__predict_false(pic_pending_percpu == NULL))
- pic_pending_percpu = percpu_alloc(sizeof(struct pic_pending));
-#endif /* __HAVE_PIC_PENDING_INTRS && MULTIPROCESSOR */
-
mutex_enter(&pic_lock);
if (irqbase == PIC_IRQBASE_ALLOC) {
irqbase = pic_lastbase;
