Plumb kmem_buckets arguments through kvmalloc_node_noprof() so it is possible to provide an API to perform kvmalloc-style allocations with a particular set of buckets. Introduce kvmalloc_buckets_node() that takes a kmem_buckets argument.
Signed-off-by: Kees Cook <[email protected]> --- Cc: Vlastimil Babka <[email protected]> Cc: Christoph Lameter <[email protected]> Cc: Pekka Enberg <[email protected]> Cc: David Rientjes <[email protected]> Cc: Joonsoo Kim <[email protected]> Cc: jvoisin <[email protected]> Cc: Andrew Morton <[email protected]> Cc: Roman Gushchin <[email protected]> Cc: Hyeonggon Yoo <[email protected]> Cc: [email protected] --- include/linux/slab.h | 19 +++++++++++++++---- lib/rhashtable.c | 2 +- mm/util.c | 13 +++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index b1165b22cc6f..8853c6eb20b4 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -799,11 +799,22 @@ static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags) #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) -extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1); -#define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) +#ifdef CONFIG_SLAB_BUCKETS +extern void *kvmalloc_buckets_node_noprof(kmem_buckets *b, size_t size, gfp_t flags, int node) + __alloc_size(2); +# define kvmalloc_node_noprof(b, size, flags, node) \ + kvmalloc_buckets_node_noprof(b, size, flags, node) +#else +extern void *kvmalloc_buckets_node_noprof(size_t size, gfp_t flags, int node) + __alloc_size(1); +# define kvmalloc_node_noprof(b, size, flags, node) \ + kvmalloc_buckets_node_noprof(size, flags, node) +#endif +#define kvmalloc_buckets_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) +#define kvmalloc_node(...) kvmalloc_buckets_node(NULL, __VA_ARGS__) #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) -#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) +#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(NULL, _size, _flags, NUMA_NO_NODE) #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) @@ -816,7 +827,7 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; - return kvmalloc_node_noprof(bytes, flags, node); + return kvmalloc_node_noprof(NULL, bytes, flags, node); } #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) diff --git a/lib/rhashtable.c b/lib/rhashtable.c index dbbed19f8fff..ef0f496e4aed 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -184,7 +184,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, static struct lock_class_key __key; tbl = alloc_hooks_tag(ht->alloc_tag, - kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets), + kvmalloc_node_noprof(NULL, struct_size(tbl, buckets, nbuckets), gfp|__GFP_ZERO, NUMA_NO_NODE)); size = nbuckets; diff --git a/mm/util.c b/mm/util.c index 80430e5ba981..53f7fc5912bd 100644 --- a/mm/util.c +++ b/mm/util.c @@ -593,9 +593,11 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, } EXPORT_SYMBOL(vm_mmap); +#ifdef CONFIG_SLAB_BUCKETS /** - * kvmalloc_node - attempt to allocate physically contiguous memory, but upon + * kvmalloc_buckets_node_noprof - attempt to allocate physically contiguous memory, but upon * failure, fall back to non-contiguous (vmalloc) allocation. + * @b: which set of kmalloc buckets to allocate from. * @size: size of the request. * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. * @node: numa node to allocate from @@ -609,7 +611,10 @@ EXPORT_SYMBOL(vm_mmap); * * Return: pointer to the allocated memory of %NULL in case of failure */ -void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) +void *kvmalloc_buckets_node_noprof(kmem_buckets *b, size_t size, gfp_t flags, int node) +#else +void *kvmalloc_buckets_node_noprof(size_t size, gfp_t flags, int node) +#endif { gfp_t kmalloc_flags = flags; void *ret; @@ -631,7 +636,7 @@ void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) kmalloc_flags &= ~__GFP_NOFAIL; } - ret = kmalloc_node_noprof(size, kmalloc_flags, node); + ret = __kmalloc_node_noprof(b, size, kmalloc_flags, node); /* * It doesn't really make sense to fallback to vmalloc for sub page @@ -660,7 +665,7 @@ void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0)); } -EXPORT_SYMBOL(kvmalloc_node_noprof); +EXPORT_SYMBOL(kvmalloc_buckets_node_noprof); /** * kvfree() - Free memory. -- 2.34.1
