Module Name: src
Committed By: riastradh
Date: Sat Feb 1 12:49:02 UTC 2020
Modified Files:
src/distrib/sets/lists/comp: mi
src/share/man/man9: Makefile percpu.9
src/sys/kern: subr_percpu.c
src/sys/sys: percpu.h
Log Message:
New function percpu_create.
Associates a constructor and destructor with the percpu. Currently
the constructor runs immediately, but in principle we could use the
same API for future CPU hotplug support.
This lets you sleep for allocation or draining users before
deallocation when setting up or tearing down a percpu -- currently we
have many abuses of percpu_foreach in tree for that purpose.
Proposed on tech-kern:
https://mail-index.NetBSD.org/tech-kern/2020/01/30/msg026036.html
To generate a diff of this commit:
cvs rdiff -u -r1.2307 -r1.2308 src/distrib/sets/lists/comp/mi
cvs rdiff -u -r1.445 -r1.446 src/share/man/man9/Makefile
cvs rdiff -u -r1.12 -r1.13 src/share/man/man9/percpu.9
cvs rdiff -u -r1.20 -r1.21 src/sys/kern/subr_percpu.c
cvs rdiff -u -r1.3 -r1.4 src/sys/sys/percpu.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/distrib/sets/lists/comp/mi
diff -u src/distrib/sets/lists/comp/mi:1.2307 src/distrib/sets/lists/comp/mi:1.2308
--- src/distrib/sets/lists/comp/mi:1.2307 Wed Jan 29 18:39:04 2020
+++ src/distrib/sets/lists/comp/mi Sat Feb 1 12:49:02 2020
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.2307 2020/01/29 18:39:04 maya Exp $
+# $NetBSD: mi,v 1.2308 2020/02/01 12:49:02 riastradh Exp $
#
# Note: don't delete entries from here - mark them as "obsolete" instead.
./etc/mtree/set.comp comp-sys-root
@@ -11570,6 +11570,7 @@
./usr/share/man/cat9/pcu_used_p.0 comp-sys-catman .cat
./usr/share/man/cat9/percpu.0 comp-sys-catman .cat
./usr/share/man/cat9/percpu_alloc.0 comp-sys-catman .cat
+./usr/share/man/cat9/percpu_create.0 comp-sys-catman .cat
./usr/share/man/cat9/percpu_foreach.0 comp-sys-catman .cat
./usr/share/man/cat9/percpu_free.0 comp-sys-catman .cat
./usr/share/man/cat9/percpu_getref.0 comp-sys-catman .cat
@@ -19498,6 +19499,7 @@
./usr/share/man/html9/pcu_used_p.html comp-sys-htmlman html
./usr/share/man/html9/percpu.html comp-sys-htmlman html
./usr/share/man/html9/percpu_alloc.html comp-sys-htmlman html
+./usr/share/man/html9/percpu_create.html comp-sys-htmlman html
./usr/share/man/html9/percpu_foreach.html comp-sys-htmlman html
./usr/share/man/html9/percpu_free.html comp-sys-htmlman html
./usr/share/man/html9/percpu_getref.html comp-sys-htmlman html
@@ -27585,6 +27587,7 @@
./usr/share/man/man9/pcu_used_p.9 comp-sys-man .man
./usr/share/man/man9/percpu.9 comp-sys-man .man
./usr/share/man/man9/percpu_alloc.9 comp-sys-man .man
+./usr/share/man/man9/percpu_create.9 comp-sys-man .man
./usr/share/man/man9/percpu_foreach.9 comp-sys-man .man
./usr/share/man/man9/percpu_free.9 comp-sys-man .man
./usr/share/man/man9/percpu_getref.9 comp-sys-man .man
Index: src/share/man/man9/Makefile
diff -u src/share/man/man9/Makefile:1.445 src/share/man/man9/Makefile:1.446
--- src/share/man/man9/Makefile:1.445 Mon Jan 20 18:38:18 2020
+++ src/share/man/man9/Makefile Sat Feb 1 12:49:02 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.445 2020/01/20 18:38:18 thorpej Exp $
+# $NetBSD: Makefile,v 1.446 2020/02/01 12:49:02 riastradh Exp $
# Makefile for section 9 (kernel function and variable) manual pages.
@@ -663,6 +663,7 @@ MLINKS+=pcmcia.9 pcmcia_function_init.9
pcmcia.9 pcmcia_cis_read_n.9 \
pcmcia.9 pcmcia_scan_cis.9
MLINKS+=percpu.9 percpu_alloc.9 \
+ percpu.9 percpu_create.9 \
percpu.9 percpu_free.9 \
percpu.9 percpu_getref.9 \
percpu.9 percpu_putref.9 \
Index: src/share/man/man9/percpu.9
diff -u src/share/man/man9/percpu.9:1.12 src/share/man/man9/percpu.9:1.13
--- src/share/man/man9/percpu.9:1.12 Wed May 31 23:54:17 2017
+++ src/share/man/man9/percpu.9 Sat Feb 1 12:49:02 2020
@@ -1,4 +1,4 @@
-.\" $NetBSD: percpu.9,v 1.12 2017/05/31 23:54:17 chs Exp $
+.\" $NetBSD: percpu.9,v 1.13 2020/02/01 12:49:02 riastradh Exp $
.\"
.\" Copyright (c) 2010 The NetBSD Foundation, Inc.
.\" All rights reserved.
@@ -27,12 +27,13 @@
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd May 31, 2017
+.Dd January 29, 2020
.Dt PERCPU 9
.Os
.Sh NAME
.Nm percpu ,
.Nm percpu_alloc ,
+.Nm percpu_create ,
.Nm percpu_free ,
.Nm percpu_getref ,
.Nm percpu_putref ,
@@ -43,6 +44,8 @@
.Vt typedef void (*percpu_callback_t)(void *, void *, struct cpu_info *);
.Ft percpu_t *
.Fn percpu_alloc "size_t size"
+.Ft percpu_t *
+.Fn percpu_create "size_t size" "percpu_callback_t ctor" "percpu_callback_t dtor" "void *arg"
.Ft void
.Fn percpu_free "percpu_t *pc" "size_t size"
.Ft void *
@@ -85,6 +88,30 @@ The storage is initialized with zeroes.
Treat this as an expensive operation.
.Fn percpu_alloc
returns a handle for the per-CPU storage.
+.It Fn percpu_create "size" "ctor" "dtor" "arg"
+Like
+.Fn percpu_alloc ,
+but before returning, for each CPU, call
+.Fn "(*ctor)" p arg ci
+in the current thread, where
+.Fa p
+is the pointer to that CPU's storage and
+.Fa ci
+is the
+.Vt "struct cpu_info *"
+for that CPU.
+Further, arrange that
+.Fn percpu_free
+will do the same with
+.Fn "(*dtor)" p arg ci .
+.Pp
+.Fa ctor
+and
+.Fa dtor
+.Em MAY
+sleep, e.g. to allocate memory or to wait for users to drain before
+deallocating memory.
+Do not rely on any particular order of iteration over the CPUs.
.It Fn percpu_free "pc" "size"
Call this in thread context to
return to the system the per-CPU storage held by
@@ -93,7 +120,9 @@ return to the system the per-CPU storage
should match the
.Fa size
passed to
-.Fn percpu_alloc .
+.Fn percpu_alloc
+or
+.Fn percpu_create .
When
.Fn percpu_free
returns,
@@ -111,6 +140,13 @@ Follow each
.Fn percpu_getref
call with a matching call to
.Fn percpu_putref .
+.Pp
+Caller
+.Em MUST NOT
+sleep after
+.Fn percpu_getref ,
+not even on an adaptive lock, before
+.Fn percpu_putref .
.It Fn percpu_putref "pc"
Indicate that the thread is finished
with the pointer returned by the matching
@@ -118,9 +154,9 @@ call to
.Fn percpu_getref .
Re-enables preemption.
.It Fn percpu_foreach "pc" "cb" "arg"
-On each CPU, for
+For each CPU, with
.Fa ci
-the corresponding
+being the corresponding
.Vt "struct cpu_info *"
and
.Fa "p"
@@ -132,12 +168,17 @@ run
.Fa "arg"
.Fa "ci"
.Fc .
-Call this in thread context.
+The call to
.Fa cb
-should be non-blocking and fast.
-Do not rely on
+runs in the current thread; use
+.Xr xcall 9
+for cross-calls to run logic on other CPUs.
+.Pp
+Must be used in thread context.
.Fa cb
-to be run on the CPUs in any particular order.
+.Em MUST NOT
+sleep except on adaptive locks, and should be fast.
+Do not rely on any particular order of iteration over the CPUs.
.El
.Sh CODE REFERENCES
The
Index: src/sys/kern/subr_percpu.c
diff -u src/sys/kern/subr_percpu.c:1.20 src/sys/kern/subr_percpu.c:1.21
--- src/sys/kern/subr_percpu.c:1.20 Thu Dec 5 03:21:08 2019
+++ src/sys/kern/subr_percpu.c Sat Feb 1 12:49:02 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_percpu.c,v 1.20 2019/12/05 03:21:08 riastradh Exp $ */
+/* $NetBSD: subr_percpu.c,v 1.21 2020/02/01 12:49:02 riastradh Exp $ */
/*-
* Copyright (c)2007,2008 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.20 2019/12/05 03:21:08 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.21 2020/02/01 12:49:02 riastradh Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -47,14 +47,12 @@ __KERNEL_RCSID(0, "$NetBSD: subr_percpu.
#define PERCPU_QCACHE_MAX 0
#define PERCPU_IMPORT_SIZE 2048
-#if defined(DIAGNOSTIC)
-#define MAGIC 0x50435055 /* "PCPU" */
-#define percpu_encrypt(pc) ((pc) ^ MAGIC)
-#define percpu_decrypt(pc) ((pc) ^ MAGIC)
-#else /* defined(DIAGNOSTIC) */
-#define percpu_encrypt(pc) (pc)
-#define percpu_decrypt(pc) (pc)
-#endif /* defined(DIAGNOSTIC) */
+struct percpu {
+ unsigned pc_offset;
+ size_t pc_size;
+ percpu_callback_t pc_dtor;
+ void *pc_cookie;
+};
static krwlock_t percpu_swap_lock __cacheline_aligned;
static kmutex_t percpu_allocation_lock __cacheline_aligned;
@@ -71,7 +69,7 @@ cpu_percpu(struct cpu_info *ci)
static unsigned int
percpu_offset(percpu_t *pc)
{
- const unsigned int off = percpu_decrypt((uintptr_t)pc);
+ const unsigned int off = pc->pc_offset;
KASSERT(off < percpu_nextoff);
return off;
@@ -253,14 +251,56 @@ percpu_init_cpu(struct cpu_info *ci)
percpu_t *
percpu_alloc(size_t size)
{
+
+ return percpu_create(size, NULL, NULL, NULL);
+}
+
+/*
+ * percpu_create: allocate percpu storage and associate ctor/dtor with it
+ *
+ * => called in thread context.
+ * => considered as an expensive and rare operation.
+ * => allocated storage is initialized by ctor, or zeros if ctor is null
+ * => percpu_free will call dtor first, if dtor is nonnull
+ * => ctor or dtor may sleep, even on allocation
+ */
+
+percpu_t *
+percpu_create(size_t size, percpu_callback_t ctor, percpu_callback_t dtor,
+ void *cookie)
+{
vmem_addr_t offset;
percpu_t *pc;
ASSERT_SLEEPABLE();
(void)vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT,
&offset);
- pc = (percpu_t *)percpu_encrypt((uintptr_t)offset);
- percpu_zero(pc, size);
+
+ pc = kmem_alloc(sizeof(*pc), KM_SLEEP);
+ pc->pc_offset = offset;
+ pc->pc_size = size;
+ pc->pc_dtor = dtor;
+ pc->pc_cookie = cookie;
+
+ if (ctor) {
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+ void *buf;
+
+ buf = kmem_alloc(size, KM_SLEEP);
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ memset(buf, 0, size);
+ (*ctor)(buf, cookie, ci);
+ percpu_traverse_enter();
+ memcpy(percpu_getptr_remote(pc, ci), buf, size);
+ percpu_traverse_exit();
+ }
+ explicit_memset(buf, 0, size);
+ kmem_free(buf, size);
+ } else {
+ percpu_zero(pc, size);
+ }
+
return pc;
}
@@ -276,7 +316,27 @@ percpu_free(percpu_t *pc, size_t size)
{
ASSERT_SLEEPABLE();
+ KASSERT(size == pc->pc_size);
+
+ if (pc->pc_dtor) {
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+ void *buf;
+
+ buf = kmem_alloc(size, KM_SLEEP);
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ percpu_traverse_enter();
+ memcpy(buf, percpu_getptr_remote(pc, ci), size);
+ explicit_memset(percpu_getptr_remote(pc, ci), 0, size);
+ percpu_traverse_exit();
+ (*pc->pc_dtor)(buf, pc->pc_cookie, ci);
+ }
+ explicit_memset(buf, 0, size);
+ kmem_free(buf, size);
+ }
+
vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size);
+ kmem_free(pc, sizeof(*pc));
}
/*
Index: src/sys/sys/percpu.h
diff -u src/sys/sys/percpu.h:1.3 src/sys/sys/percpu.h:1.4
--- src/sys/sys/percpu.h:1.3 Wed Apr 9 05:11:20 2008
+++ src/sys/sys/percpu.h Sat Feb 1 12:49:02 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: percpu.h,v 1.3 2008/04/09 05:11:20 thorpej Exp $ */
+/* $NetBSD: percpu.h,v 1.4 2020/02/01 12:49:02 riastradh Exp $ */
/*-
* Copyright (c)2007,2008 YAMAMOTO Takashi,
@@ -41,6 +41,8 @@ void percpu_putref(percpu_t *);
typedef void (*percpu_callback_t)(void *, void *, struct cpu_info *);
void percpu_foreach(percpu_t *, percpu_callback_t, void *);
+percpu_t *percpu_create(size_t, percpu_callback_t, percpu_callback_t, void *);
+
/* low-level api; don't use unless necessary */
void percpu_traverse_enter(void);
void percpu_traverse_exit(void);
