From: David Stevens <[email protected]> For vmap stacks not immediately released into the stack cache, free them in a workqueue instead of via call_rcu(). In an RCU context, vfree already schedules the actual freeing on the per-cpu system workqueue, so this change only affects when exactly the second attempt to put the stack into the stack cache occurs.
Moving freeing to a workqueue will allow for freeing partially populated stacks in a sleepable context (for remove_vm_area), rather than relying on vfree dispatching to a workqueue via vfree_atomic. Signed-off-by: David Stevens <[email protected]> [Use system_percpu_wq instead of system_wq as it is deprecated and update commit msg] Signed-off-by: Mostafa Saleh <[email protected]> --- kernel/fork.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 3f1c46d8f7ec..ca8e68882316 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -208,7 +208,7 @@ static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); #define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN) struct vm_stack { - struct rcu_head rcu; + struct rcu_work work; struct vm_struct *stack_vm_area; }; @@ -287,9 +287,9 @@ static inline void free_vmap_stack(struct vm_struct *vm_area) vfree(vm_area->addr); } -static void thread_stack_free_rcu(struct rcu_head *rh) +static void thread_stack_free_work(struct work_struct *work) { - struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu); + struct vm_stack *vm_stack = container_of(to_rcu_work(work), struct vm_stack, work); struct vm_struct *vm_area = vm_stack->stack_vm_area; if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) @@ -308,7 +308,8 @@ static void thread_stack_delayed_free(struct task_struct *tsk) vm_stack = tsk->stack + THREAD_SIZE - sizeof(*vm_stack); vm_stack->stack_vm_area = tsk->stack_vm_area; - call_rcu(&vm_stack->rcu, thread_stack_free_rcu); + INIT_RCU_WORK(&vm_stack->work, thread_stack_free_work); + queue_rcu_work(system_percpu_wq, &vm_stack->work); } static int free_vm_stack_cache(unsigned int cpu) -- 2.56.0.rc1.315.gc6ed9934b7-goog
