Currently, k[v]free_rcu() cannot be called in unknown context since
it could lead to a deadlock when called in the middle of k[v]free_rcu().
Make users' lives easier by introducing kfree_rcu_nolock() variant,
now that kfree_rcu_sheaf() is available on PREEMPT_RT and
__kfree_rcu_sheaf() handles unknown context.
When sheaves path fails, kfree_rcu_nolock() falls back to
defer_kfree_rcu() that uses an irq work to free the object via
kvfree_call_rcu(). In most cases, the sheaves path is expected to
succeed and therefore it's unnecessary to introduce additional
complexity to the existing kvfree_rcu batching by teaching it
how to handle unknown context.
Since defer_kfree_rcu() can be called on caches without sheaves, move
deferred_work_barrier() and rcu_barrier() outside the branch in
kvfree_rcu_barrier_on_cache().
Now that deferred kvfree_rcu objects are submitted to kvfree_call_rcu()
after deferred_work_barrier() and may end up in RCU sheaves,
deferred_work_barrier() must be invoked before
flush_rcu_sheaves_on_cache().
Since the RCU sheaf path has not been used on !KVFREE_RCU_BATCHED
kernels, always fall back when kvfree_rcu() is not batched, for
consistency. kvfree_rcu_barrier{,_on_cache()}() on
!KVFREE_RCU_BATCHED are moved to mm/slab_common.c to invoke
deferred_work_barrier() before rcu_barrier().
Signed-off-by: Harry Yoo (Oracle) <[email protected]>
---
include/linux/rcupdate.h | 22 ++++++++++++++++++++++
include/linux/slab.h | 16 +++-------------
mm/slab_common.c | 47 +++++++++++++++++++++++++++++++++++++++++++++--
mm/slub.c | 28 ++++++++++++++++++++++++++--
4 files changed, 96 insertions(+), 17 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index ef5bb6981133..aede77bd0387 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1099,6 +1099,7 @@ static inline void rcu_read_unlock_migrate(void)
* In mm/slab_common.c, no suitable header to include here.
*/
void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr);
+void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr);
/*
* The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
@@ -1124,6 +1125,27 @@ do {
\
kvfree_call_rcu(NULL, (void *) (___p)); \
} while (0)
+/**
+ * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any
context.
+ * @ptr: pointer to kfree for double-argument invocations.
+ * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr.
+ *
+ * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects
+ * without any deferred processing, but may still defer freeing.
+ * Large kmalloc and vmalloc objects are always deferred.
+ *
+ * kfree_rcu_nolock() supports 2-arg variant only.
+ */
+#define kfree_rcu_nolock(ptr, kvrhf)
\
+do {
\
+ typeof (ptr) ___p = (ptr);
\
+
\
+ if (___p) {
\
+ BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);
\
+ kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p));
\
+ }
\
+} while (0)
+
/*
* Place this after a lock-acquisition primitive to guarantee that
* an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies
diff --git a/include/linux/slab.h b/include/linux/slab.h
index e1915026e030..cda126def67a 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -1430,25 +1430,15 @@ extern void kvfree_sensitive(const void *addr, size_t
len);
unsigned int kmem_cache_size(struct kmem_cache *s);
#ifndef CONFIG_KVFREE_RCU_BATCHED
-static inline void kvfree_rcu_barrier(void)
-{
- rcu_barrier();
-}
-
-static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
-{
- rcu_barrier();
-}
-
static inline void kfree_rcu_scheduler_running(void) { }
#else
+void kfree_rcu_scheduler_running(void);
+#endif
+
void kvfree_rcu_barrier(void);
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
-void kfree_rcu_scheduler_running(void);
-#endif
-
/**
* kmalloc_size_roundup - Report allocation bucket size for the given size
*
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 691ce6db4519..b8f21eabcccc 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -1296,6 +1296,33 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
EXPORT_TRACEPOINT_SYMBOL(kfree);
EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
+void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr)
+{
+ struct slab *slab;
+
+ if (!IS_ENABLED(CONFIG_KVFREE_RCU_BATCHED))
+ goto fallback;
+
+ if (unlikely(is_vmalloc_addr(ptr)))
+ goto fallback;
+
+ slab = virt_to_slab(ptr);
+ if (unlikely(!slab))
+ goto fallback;
+
+ if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) !=
numa_mem_id()))
+ goto fallback;
+
+ if (unlikely(!__kfree_rcu_sheaf(slab->slab_cache, ptr,
SLAB_FREE_NOLOCK)))
+ goto fallback;
+
+ return;
+
+fallback:
+ defer_kfree_rcu(head);
+}
+EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock);
+
#ifndef CONFIG_KVFREE_RCU_BATCHED
void kvfree_call_rcu(struct rcu_head *head, void *ptr)
@@ -1313,6 +1340,20 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr)
}
EXPORT_SYMBOL_GPL(kvfree_call_rcu);
+void kvfree_rcu_barrier(void)
+{
+ deferred_work_barrier();
+ rcu_barrier();
+}
+EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
+
+void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
+{
+ deferred_work_barrier();
+ rcu_barrier();
+}
+EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache);
+
void __init kvfree_rcu_init(void)
{
}
@@ -2149,14 +2190,16 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
*/
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
+ /* kfree_rcu_nolock() might have deferred frees even without sheaves */
+ deferred_work_barrier();
+
if (cache_has_sheaves(s)) {
cpus_read_lock();
flush_rcu_sheaves_on_cache(s);
cpus_read_unlock();
- deferred_work_barrier();
- rcu_barrier();
}
+ rcu_barrier();
__kvfree_rcu_barrier();
}
EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache);
diff --git a/mm/slub.c b/mm/slub.c
index 774104267f3c..6e5aa38e7569 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4099,6 +4099,7 @@ static void flush_all(struct kmem_cache *s)
struct deferred_percpu_work {
struct llist_head objects;
+ struct llist_head objects_by_rcu;
struct llist_head rcu_sheaves;
struct irq_work work;
};
@@ -4107,6 +4108,7 @@ static void deferred_percpu_work_fn(struct irq_work
*work);
static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
.objects = LLIST_HEAD_INIT(objects),
+ .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
.rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
};
@@ -4170,6 +4172,8 @@ void flush_all_rcu_sheaves(void)
{
struct kmem_cache *s;
+ deferred_work_barrier();
+
cpus_read_lock();
mutex_lock(&slab_mutex);
@@ -4182,7 +4186,6 @@ void flush_all_rcu_sheaves(void)
mutex_unlock(&slab_mutex);
cpus_read_unlock();
- deferred_work_barrier();
rcu_barrier();
}
@@ -6407,13 +6410,14 @@ static void free_to_pcs_bulk(struct kmem_cache *s,
size_t size, void **p)
static void deferred_percpu_work_fn(struct irq_work *work)
{
struct deferred_percpu_work *dpw;
- struct llist_head *objs, *rcu_sheaves;
+ struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
struct llist_node *llnode, *pos, *t;
struct slab_sheaf *sheaf, *next;
dpw = container_of(work, struct deferred_percpu_work, work);
rcu_sheaves = &dpw->rcu_sheaves;
objs = &dpw->objects;
+ objs_by_rcu = &dpw->objects_by_rcu;
llnode = llist_del_all(objs);
llist_for_each_safe(pos, t, llnode) {
@@ -6438,6 +6442,14 @@ static void deferred_percpu_work_fn(struct irq_work
*work)
stat(s, FREE_SLOWPATH);
}
+ llnode = llist_del_all(objs_by_rcu);
+ llist_for_each_safe(pos, t, llnode) {
+ void *head = pos;
+ void *objp = kvmalloc_obj_start_addr(head);
+
+ kvfree_call_rcu(head, objp);
+ }
+
llnode = llist_del_all(rcu_sheaves);
llist_for_each_entry_safe(sheaf, next, llnode, llnode)
call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
@@ -6456,6 +6468,18 @@ static void defer_free(struct kmem_cache *s, void *head)
irq_work_queue(&dpw->work);
}
+void defer_kfree_rcu(struct kvfree_rcu_head *head)
+{
+ struct deferred_percpu_work *dpw;
+
+ guard(preempt)();
+
+ dpw = this_cpu_ptr(&deferred_percpu_work);
+ if (llist_add((struct llist_node *)head, &dpw->objects_by_rcu))
+ irq_work_queue(&dpw->work);
+}
+
+/* Must be called before flush_rcu_sheaves_on_cache() */
void deferred_work_barrier(void)
{
int cpu;
--
2.53.0
