From: Pasha Tatashin <[email protected]> In preparation for the partially populated stacks, separate out the __vmalloc_node and vfree calls from the vmap based stack allocations.
Signed-off-by: Pasha Tatashin <[email protected]> [Fix a bug in original patch: free_vmap_stack(vm_area->addr)] Signed-off-by: Linus Walleij <[email protected]> [Add missing free_vmap_stack conversion, fix typos, rebase] Signed-off-by: David Stevens <[email protected]> [Resolved conflict in alloc_thread_stack_node(), update commit msg] Signed-off-by: Mostafa Saleh <[email protected]> --- kernel/fork.c | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 473878d7f62f..3f1c46d8f7ec 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -272,6 +272,21 @@ static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area) return false; } +static inline struct vm_struct *alloc_vmap_stack(int node) +{ + void *stack; + + stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, GFP_VMAP_STACK, + node, __builtin_return_address(0)); + + return stack ? find_vm_area(stack) : NULL; +} + +static inline void free_vmap_stack(struct vm_struct *vm_area) +{ + vfree(vm_area->addr); +} + static void thread_stack_free_rcu(struct rcu_head *rh) { struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu); @@ -280,7 +295,7 @@ static void thread_stack_free_rcu(struct rcu_head *rh) if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) return; - vfree(vm_area->addr); + free_vmap_stack(vm_area); } static void thread_stack_delayed_free(struct task_struct *tsk) @@ -307,7 +322,7 @@ static int free_vm_stack_cache(unsigned int cpu) if (!vm_area) continue; - vfree(vm_area->addr); + free_vmap_stack(vm_area); cached_vm_stack_areas[i] = NULL; } @@ -336,14 +351,14 @@ static int memcg_charge_kernel_stack(struct vm_struct *vm_area) static int alloc_thread_stack_node(struct task_struct *tsk, int node) { struct vm_struct *vm_area; - void *stack; vm_area = alloc_thread_stack_node_from_cache(tsk, node); if (vm_area) { unsigned long offset = 0; + void *stack; if (memcg_charge_kernel_stack(vm_area)) { - vfree(vm_area->addr); + free_vmap_stack(vm_area); return -ENOMEM; } @@ -364,15 +379,12 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) return 0; } - stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, - GFP_VMAP_STACK, - node, __builtin_return_address(0)); - if (!stack) + vm_area = alloc_vmap_stack(node); + if (!vm_area) return -ENOMEM; - vm_area = find_vm_area(stack); if (memcg_charge_kernel_stack(vm_area)) { - vfree(stack); + free_vmap_stack(vm_area); return -ENOMEM; } /* @@ -381,8 +393,7 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) * so cache the vm_struct. */ tsk->stack_vm_area = vm_area; - stack = kasan_reset_tag(stack); - tsk->stack = stack; + tsk->stack = kasan_reset_tag(vm_area->addr); return 0; } -- 2.56.0.rc1.315.gc6ed9934b7-goog
