Module Name: src
Committed By: riastradh
Date: Sun Dec 19 01:20:46 UTC 2021
Modified Files:
src/sys/external/bsd/drm2/include/linux: rcupdate.h
src/sys/external/bsd/drm2/linux: linux_rcu.c
Log Message:
Implement kfree_rcu.
To generate a diff of this commit:
cvs rdiff -u -r1.11 -r1.12 src/sys/external/bsd/drm2/include/linux/rcupdate.h
cvs rdiff -u -r1.7 -r1.8 src/sys/external/bsd/drm2/linux/linux_rcu.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/external/bsd/drm2/include/linux/rcupdate.h
diff -u src/sys/external/bsd/drm2/include/linux/rcupdate.h:1.11 src/sys/external/bsd/drm2/include/linux/rcupdate.h:1.12
--- src/sys/external/bsd/drm2/include/linux/rcupdate.h:1.11 Sun Dec 19 01:19:45 2021
+++ src/sys/external/bsd/drm2/include/linux/rcupdate.h Sun Dec 19 01:20:45 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: rcupdate.h,v 1.11 2021/12/19 01:19:45 riastradh Exp $ */
+/* $NetBSD: rcupdate.h,v 1.12 2021/12/19 01:20:45 riastradh Exp $ */
/*-
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -67,10 +67,14 @@
#define rcu_pointer_handoff(P) (P)
struct rcu_head {
- void (*rcuh_callback)(struct rcu_head *);
+ union {
+ void (*callback)(struct rcu_head *);
+ void *obj;
+ } rcuh_u;
struct rcu_head *rcuh_next;
};
+#define _kfree_rcu linux__kfree_rcu
#define call_rcu linux_call_rcu
#define rcu_barrier linux_rcu_barrier
#define synchronize_rcu linux_synchronize_rcu
@@ -82,6 +86,8 @@ void call_rcu(struct rcu_head *, void (*
void rcu_barrier(void);
void synchronize_rcu(void);
+void _kfree_rcu(struct rcu_head *, void *);
+
static inline void
rcu_read_lock(void)
{
@@ -98,4 +104,7 @@ rcu_read_unlock(void)
kpreempt_enable();
}
+#define kfree_rcu(P, F) \
+ _kfree_rcu(&(P)->F, (P))
+
#endif /* _LINUX_RCUPDATE_H_ */
Index: src/sys/external/bsd/drm2/linux/linux_rcu.c
diff -u src/sys/external/bsd/drm2/linux/linux_rcu.c:1.7 src/sys/external/bsd/drm2/linux/linux_rcu.c:1.8
--- src/sys/external/bsd/drm2/linux/linux_rcu.c:1.7 Sun Dec 19 01:19:52 2021
+++ src/sys/external/bsd/drm2/linux/linux_rcu.c Sun Dec 19 01:20:45 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: linux_rcu.c,v 1.7 2021/12/19 01:19:52 riastradh Exp $ */
+/* $NetBSD: linux_rcu.c,v 1.8 2021/12/19 01:20:45 riastradh Exp $ */
/*-
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: linux_rcu.c,v 1.7 2021/12/19 01:19:52 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: linux_rcu.c,v 1.8 2021/12/19 01:20:45 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -42,6 +42,7 @@ __KERNEL_RCSID(0, "$NetBSD: linux_rcu.c,
#include <sys/xcall.h>
#include <linux/rcupdate.h>
+#include <linux/slab.h>
SDT_PROBE_DEFINE0(sdt, linux, rcu, synchronize__start);
SDT_PROBE_DEFINE1(sdt, linux, rcu, synchronize__cpu, "unsigned"/*cpu*/);
@@ -54,11 +55,18 @@ SDT_PROBE_DEFINE2(sdt, linux, rcu, call_
"struct rcu_head *"/*head*/, "void (*)(struct rcu_head *)"/*callback*/);
SDT_PROBE_DEFINE2(sdt, linux, rcu, call__done,
"struct rcu_head *"/*head*/, "void (*)(struct rcu_head *)"/*callback*/);
+SDT_PROBE_DEFINE2(sdt, linux, rcu, kfree__queue,
+ "struct rcu_head *"/*head*/, "void *"/*obj*/);
+SDT_PROBE_DEFINE2(sdt, linux, rcu, kfree__free,
+ "struct rcu_head *"/*head*/, "void *"/*obj*/);
+SDT_PROBE_DEFINE2(sdt, linux, rcu, kfree__done,
+ "struct rcu_head *"/*head*/, "void *"/*obj*/);
static struct {
kmutex_t lock;
kcondvar_t cv;
- struct rcu_head *first;
+ struct rcu_head *first_callback;
+ struct rcu_head *first_kfree;
struct lwp *lwp;
uint64_t gen;
bool dying;
@@ -101,7 +109,7 @@ rcu_barrier(void)
SDT_PROBE0(sdt, linux, rcu, barrier__start);
mutex_enter(&gc.lock);
- if (gc.first != NULL) {
+ if (gc.first_callback != NULL || gc.first_kfree != NULL) {
gen = gc.gen;
do {
cv_wait(&gc.cv, &gc.lock);
@@ -121,68 +129,109 @@ void
call_rcu(struct rcu_head *head, void (*callback)(struct rcu_head *))
{
- head->rcuh_callback = callback;
+ head->rcuh_u.callback = callback;
mutex_enter(&gc.lock);
- head->rcuh_next = gc.first;
- gc.first = head;
+ head->rcuh_next = gc.first_callback;
+ gc.first_callback = head;
cv_broadcast(&gc.cv);
SDT_PROBE2(sdt, linux, rcu, call__queue, head, callback);
mutex_exit(&gc.lock);
}
+/*
+ * _kfree_rcu(head, obj)
+ *
+ * kfree_rcu helper: schedule kfree(obj) using head for storage.
+ */
+void
+_kfree_rcu(struct rcu_head *head, void *obj)
+{
+
+ head->rcuh_u.obj = obj;
+
+ mutex_enter(&gc.lock);
+ head->rcuh_next = gc.first_kfree;
+ gc.first_kfree = head;
+ cv_broadcast(&gc.cv);
+ SDT_PROBE2(sdt, linux, rcu, kfree__queue, head, obj);
+ mutex_exit(&gc.lock);
+}
+
static void
gc_thread(void *cookie)
{
- struct rcu_head *head, *next;
- void (*callback)(struct rcu_head *);
+ struct rcu_head *head_callback, *head_kfree, *head, *next;
mutex_enter(&gc.lock);
for (;;) {
- /* Wait for a task or death notice. */
- while ((head = gc.first) == NULL && !gc.dying)
+ /* Start with no work. */
+ bool work = false;
+
+ /* Grab the list of callbacks. */
+ if ((head_callback = gc.first_callback) != NULL) {
+ gc.first_callback = NULL;
+ work = true;
+ }
+
+ /* Grab the list of objects to kfree. */
+ if ((head_kfree = gc.first_kfree) != NULL) {
+ gc.first_kfree = NULL;
+ work = true;
+ }
+
+ /*
+ * If no work, then either stop, if we're dying, or
+ * wait for work, if not.
+ */
+ if (!work) {
+ if (gc.dying)
+ break;
cv_wait(&gc.cv, &gc.lock);
+ continue;
+ }
+
+ /* We have work to do. Drop the lock to do it. */
+ mutex_exit(&gc.lock);
- /* If we got a list of callbacks, run them. */
- if (head != NULL) {
- gc.first = NULL; /* mine */
- mutex_exit(&gc.lock);
-
- /* Wait for activity on all CPUs. */
- synchronize_rcu();
-
- /* It is now safe to call the callbacks. */
- for (; head != NULL; head = next) {
- next = head->rcuh_next;
- callback = head->rcuh_callback;
- SDT_PROBE2(sdt, linux, rcu, call__run,
- head, callback);
- (*callback)(head);
- /*
- * Can't dereference head or invoke
- * callback after this point.
- */
- SDT_PROBE2(sdt, linux, rcu, call__done,
- head, callback);
- }
-
- mutex_enter(&gc.lock);
- gc.gen++; /* done running */
- cv_broadcast(&gc.cv); /* notify rcu_barrier */
+ /* Wait for activity on all CPUs. */
+ synchronize_rcu();
+ /* Call the callbacks. */
+ for (head = head_callback; head != NULL; head = next) {
+ void (*callback)(struct rcu_head *) =
+ head->rcuh_u.callback;
+ next = head->rcuh_next;
+ SDT_PROBE2(sdt, linux, rcu, call__run,
+ head, callback);
+ (*callback)(head);
/*
- * Go back to the top and get more work before
- * deciding whether to stop so that we
- * guarantee to run all callbacks.
+ * Can't dereference head or invoke
+ * callback after this point.
*/
- continue;
+ SDT_PROBE2(sdt, linux, rcu, call__done,
+ head, callback);
}
- /* If we're asked to close shop, do so. */
- if (gc.dying)
- break;
+ /* Free the objects to kfree. */
+ for (head = head_kfree; head != NULL; head = next) {
+ void *obj = head->rcuh_u.obj;
+ next = head->rcuh_next;
+ SDT_PROBE2(sdt, linux, rcu, kfree__free, head, obj);
+ kfree(obj);
+ /* Can't dereference head or obj after this point. */
+ SDT_PROBE2(sdt, linux, rcu, kfree__done, head, obj);
+ }
+
+ /* Return to the lock. */
+ mutex_enter(&gc.lock);
+
+ /* Finished a batch of work. Notify rcu_barrier. */
+ gc.gen++;
+ cv_broadcast(&gc.cv);
}
- KASSERT(gc.first == NULL);
+ KASSERT(gc.first_callback == NULL);
+ KASSERT(gc.first_kfree == NULL);
mutex_exit(&gc.lock);
kthread_exit(0);
@@ -195,7 +244,8 @@ linux_rcu_gc_init(void)
mutex_init(&gc.lock, MUTEX_DEFAULT, IPL_VM);
cv_init(&gc.cv, "lnxrcugc");
- gc.first = NULL;
+ gc.first_callback = NULL;
+ gc.first_kfree = NULL;
gc.gen = 0;
gc.dying = false;
@@ -224,7 +274,8 @@ linux_rcu_gc_fini(void)
kthread_join(gc.lwp);
gc.lwp = NULL;
- KASSERT(gc.first == NULL);
+ KASSERT(gc.first_callback == NULL);
+ KASSERT(gc.first_kfree == NULL);
cv_destroy(&gc.cv);
mutex_destroy(&gc.lock);
}
