From: Thomas Gleixner <t...@linutronix.de>
Make SLUB RT aware and remove the restriction in Kconfig.
Signed-off-by: Thomas Gleixner <t...@linutronix.de>
[bigeasy@linutronix: fix a few conflicts]
Signed-off-by: Sebastian Andrzej Siewior <bige...@linutronix.de>
---
include/linux/slub_def.h | 2 +-
init/Kconfig | 1 -
mm/slub.c | 115 +++++++++++++++++++++++++++++++++++-----------
3 files changed, 90 insertions(+), 28 deletions(-)
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index c2f8c8b..f0a69f5 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -55,7 +55,7 @@ struct kmem_cache_cpu {
};
struct kmem_cache_node {
- spinlock_t list_lock; /* Protect partial list and nr_partial */
+ raw_spinlock_t list_lock; /* Protect partial list and nr_partial
*/
unsigned long nr_partial;
struct list_head partial;
#ifdef CONFIG_SLUB_DEBUG
diff --git a/init/Kconfig b/init/Kconfig
index 87afda5..5390b4b 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1266,7 +1266,6 @@ config SLAB
config SLUB
bool "SLUB (Unqueued Allocator)"
- depends on !PREEMPT_RT_FULL
help
SLUB is a slab allocator that minimizes cache line usage
instead of managing queues of cached objects (SLAB approach).
diff --git a/mm/slub.c b/mm/slub.c
index f788e7e..624deaa 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1266,6 +1266,12 @@ static inline void slab_free_hook(struct kmem_cache *s,
void *x) {}
#endif /* CONFIG_SLUB_DEBUG */
+struct slub_free_list {
+ raw_spinlock_t lock;
+ struct list_head list;
+};
+static DEFINE_PER_CPU(struct slub_free_list, slub_free_list);
+
/*
* Slab allocation and freeing
*/
@@ -1290,7 +1296,11 @@ static struct page *allocate_slab(struct kmem_cache *s,
gfp_t flags, int node)
flags &= gfp_allowed_mask;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ if (system_state == SYSTEM_RUNNING)
+#else
if (flags & __GFP_WAIT)
+#endif
local_irq_enable();
flags |= s->allocflags;
@@ -1314,7 +1324,11 @@ static struct page *allocate_slab(struct kmem_cache *s,
gfp_t flags, int node)
stat(s, ORDER_FALLBACK);
}
+#ifdef CONFIG_PREEMPT_RT_FULL
+ if (system_state == SYSTEM_RUNNING)
+#else
if (flags & __GFP_WAIT)
+#endif
local_irq_disable();
if (!page)
@@ -1420,6 +1434,16 @@ static void __free_slab(struct kmem_cache *s, struct
page *page)
__free_pages(page, order);
}
+static void free_delayed(struct kmem_cache *s, struct list_head *h)
+{
+ while(!list_empty(h)) {
+ struct page *page = list_first_entry(h, struct page, lru);
+
+ list_del(&page->lru);
+ __free_slab(s, page);
+ }
+}
+
#define need_reserve_slab_rcu \
(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
@@ -1454,6 +1478,12 @@ static void free_slab(struct kmem_cache *s, struct page
*page)
}
call_rcu(head, rcu_free_slab);
+ } else if (irqs_disabled()) {
+ struct slub_free_list *f = &__get_cpu_var(slub_free_list);
+
+ raw_spin_lock(&f->lock);
+ list_add(&page->lru, &f->list);
+ raw_spin_unlock(&f->lock);
} else
__free_slab(s, page);
}
@@ -1553,7 +1583,7 @@ static void *get_partial_node(struct kmem_cache *s,
if (!n || !n->nr_partial)
return NULL;
- spin_lock(&n->list_lock);
+ raw_spin_lock(&n->list_lock);
list_for_each_entry_safe(page, page2, &n->partial, lru) {
void *t = acquire_slab(s, n, page, object == NULL);
int available;
@@ -1575,7 +1605,7 @@ static void *get_partial_node(struct kmem_cache *s,
break;
}
- spin_unlock(&n->list_lock);
+ raw_spin_unlock(&n->list_lock);
return object;
}
@@ -1824,7 +1854,7 @@ static void deactivate_slab(struct kmem_cache *s, struct
kmem_cache_cpu *c)
* that acquire_slab() will see a slab page that
* is frozen
*/
- spin_lock(&n->list_lock);
+ raw_spin_lock(&n->list_lock);
}
} else {
m = M_FULL;
@@ -1835,7 +1865,7 @@ static void deactivate_slab(struct kmem_cache *s, struct
kmem_cache_cpu *c)
* slabs from diagnostic functions will not see
* any frozen slabs.
*/
- spin_lock(&n->list_lock);
+ raw_spin_lock(&n->list_lock);
}
}
@@ -1870,7 +1900,7 @@ static void deactivate_slab(struct kmem_cache *s, struct
kmem_cache_cpu *c)
goto redo;
if (lock)
- spin_unlock(&n->list_lock);
+ raw_spin_unlock(&n->list_lock);
if (m == M_FREE) {
stat(s, DEACTIVATE_EMPTY);
@@ -1919,10 +1949,10 @@ static void unfreeze_partials(struct kmem_cache *s,
m = M_PARTIAL;
if (n != n2) {
if (n)
- spin_unlock(&n->list_lock);
+ raw_spin_unlock(&n->list_lock);
n = n2;
- spin_lock(&n->list_lock);
+ raw_spin_lock(&n->list_lock);
}
}
@@ -1951,7 +1981,7 @@ static void unfreeze_partials(struct kmem_cache *s,
}
if (n)
- spin_unlock(&n->list_lock);
+ raw_spin_unlock(&n->list_lock);
while (discard_page) {
page = discard_page;
@@ -1972,7 +2002,7 @@ static void unfreeze_partials(struct kmem_cache *s,
* If we did not find a slot then simply move all the partials to the
* per node partial list.
*/
-int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
+static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
{
struct page *oldpage;
int pages;
@@ -1987,6 +2017,8 @@ int put_cpu_partial(struct kmem_cache *s, struct page
*page, int drain)
pobjects = oldpage->pobjects;
pages = oldpage->pages;
if (drain && pobjects > s->cpu_partial) {
+ LIST_HEAD(tofree);
+ struct slub_free_list *f;
unsigned long flags;
/*
* partial array is full. Move the existing
@@ -1994,7 +2026,12 @@ int put_cpu_partial(struct kmem_cache *s, struct page
*page, int drain)
*/
local_irq_save(flags);
unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
+ f = &__get_cpu_var(slub_free_list);
+ raw_spin_lock(&f->lock);
+ list_splice_init(&f->list, &tofree);
+ raw_spin_unlock(&f->lock);
local_irq_restore(flags);
+ free_delayed(s, &tofree);
pobjects = 0;
pages = 0;
stat(s, CPU_PARTIAL_DRAIN);
@@ -2052,7 +2089,22 @@ static bool has_cpu_slab(int cpu, void *info)
static void flush_all(struct kmem_cache *s)
{
+ LIST_HEAD(tofree);
+ int cpu;
+
on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
+ for_each_online_cpu(cpu) {
+ struct slub_free_list *f;
+
+ if (!has_cpu_slab(cpu, s))
+ continue;
+
+ f = &per_cpu(slub_free_list, cpu);
+ raw_spin_lock_irq(&f->lock);
+ list_splice_init(&f->list, &tofree);
+ raw_spin_unlock_irq(&f->lock);
+ free_delayed(s, &tofree);
+ }
}
/*
@@ -2080,10 +2132,10 @@ static unsigned long count_partial(struct
kmem_cache_node *n,
unsigned long x = 0;
struct page *page;
- spin_lock_irqsave(&n->list_lock, flags);
+ raw_spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry(page, &n->partial, lru)
x += get_count(page);
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
return x;
}
@@ -2210,6 +2262,8 @@ static inline void *get_freelist(struct kmem_cache *s,
struct page *page)
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
unsigned long addr, struct kmem_cache_cpu *c)
{
+ struct slub_free_list *f;
+ LIST_HEAD(tofree);
void **object;
unsigned long flags;
@@ -2252,7 +2306,13 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t
gfpflags, int node,
load_freelist:
c->freelist = get_freepointer(s, object);
c->tid = next_tid(c->tid);
+out:
+ f = &__get_cpu_var(slub_free_list);
+ raw_spin_lock(&f->lock);
+ list_splice_init(&f->list, &tofree);
+ raw_spin_unlock(&f->lock);
local_irq_restore(flags);
+ free_delayed(s, &tofree);
return object;
new_slab:
@@ -2277,8 +2337,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t
gfpflags, int node,
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
slab_out_of_memory(s, gfpflags, node);
- local_irq_restore(flags);
- return NULL;
+ goto out;
}
}
@@ -2292,8 +2351,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t
gfpflags, int node,
c->freelist = get_freepointer(s, object);
deactivate_slab(s, c);
c->node = NUMA_NO_NODE;
- local_irq_restore(flags);
- return object;
+ goto out;
}
/*
@@ -2488,7 +2546,7 @@ static void __slab_free(struct kmem_cache *s, struct page
*page,
* Otherwise the list_lock will synchronize with
* other processors updating the list of slabs.
*/
- spin_lock_irqsave(&n->list_lock, flags);
+ raw_spin_lock_irqsave(&n->list_lock, flags);
}
}
@@ -2538,7 +2596,7 @@ static void __slab_free(struct kmem_cache *s, struct page
*page,
stat(s, FREE_ADD_PARTIAL);
}
}
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
return;
slab_empty:
@@ -2552,7 +2610,7 @@ static void __slab_free(struct kmem_cache *s, struct page
*page,
/* Slab must be on the full list */
remove_full(s, page);
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
stat(s, FREE_SLAB);
discard_slab(s, page);
}
@@ -2782,7 +2840,7 @@ static void
init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
{
n->nr_partial = 0;
- spin_lock_init(&n->list_lock);
+ raw_spin_lock_init(&n->list_lock);
INIT_LIST_HEAD(&n->partial);
#ifdef CONFIG_SLUB_DEBUG
atomic_long_set(&n->nr_slabs, 0);
@@ -3525,7 +3583,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
for (i = 0; i < objects; i++)
INIT_LIST_HEAD(slabs_by_inuse + i);
- spin_lock_irqsave(&n->list_lock, flags);
+ raw_spin_lock_irqsave(&n->list_lock, flags);
/*
* Build lists indexed by the items in use in each slab.
@@ -3546,7 +3604,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
for (i = objects - 1; i > 0; i--)
list_splice(slabs_by_inuse + i, n->partial.prev);
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
/* Release empty slabs */
list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
@@ -3712,10 +3770,15 @@ void __init kmem_cache_init(void)
int i;
int caches = 0;
struct kmem_cache *temp_kmem_cache;
- int order;
+ int order, cpu;
struct kmem_cache *temp_kmem_cache_node;
unsigned long kmalloc_size;
+ for_each_possible_cpu(cpu) {
+ raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock);
+ INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list);
+ }
+
if (debug_guardpage_minorder())
slub_max_order = 0;
@@ -4139,7 +4202,7 @@ static int validate_slab_node(struct kmem_cache *s,
struct page *page;
unsigned long flags;
- spin_lock_irqsave(&n->list_lock, flags);
+ raw_spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry(page, &n->partial, lru) {
validate_slab_slab(s, page, map);
@@ -4162,7 +4225,7 @@ static int validate_slab_node(struct kmem_cache *s,
atomic_long_read(&n->nr_slabs));
out:
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
return count;
}
@@ -4352,12 +4415,12 @@ static int list_locations(struct kmem_cache *s, char
*buf,
if (!atomic_long_read(&n->nr_slabs))
continue;
- spin_lock_irqsave(&n->list_lock, flags);
+ raw_spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry(page, &n->partial, lru)
process_slab(&t, s, page, alloc, map);
list_for_each_entry(page, &n->full, lru)
process_slab(&t, s, page, alloc, map);
- spin_unlock_irqrestore(&n->list_lock, flags);
+ raw_spin_unlock_irqrestore(&n->list_lock, flags);
}
for (i = 0; i < t.count; i++) {
--
1.7.10.4
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
