Make a mempool available for allocating bvecq structs. Use it automatically if anything other than GFP_KERNEL (with GFP_ZONEMASK flags masked off) is specified. Reading from a file can use GFP_KERNEL as the failure mode is straightforward and the same for DIO reads and writes. When it comes to writeback, however, the writeback_iter() API does not permit temporary failure, including ENOMEM, if WB_SYNC_ALL is set and the caller must process all folios to completion. (I'm not sure if EINTR constitutes an acceptable failure).
Signed-off-by: David Howells <[email protected]> cc: Paulo Alcantara <[email protected]> cc: Matthew Wilcox <[email protected]> cc: Christoph Hellwig <[email protected]> cc: [email protected] cc: [email protected] --- fs/netfs/bvecq.c | 39 +++++++++++++++++++++++++++++++++------ fs/netfs/internal.h | 1 + fs/netfs/main.c | 7 +++++++ include/linux/bvecq.h | 3 +++ 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/fs/netfs/bvecq.c b/fs/netfs/bvecq.c index 6905c84ea351..7b0aedafeda9 100644 --- a/fs/netfs/bvecq.c +++ b/fs/netfs/bvecq.c @@ -44,8 +44,9 @@ EXPORT_SYMBOL(bvecq_dump); * * Allocate a single bvecq node and initialise the header. A number of inline * slots are also allocated, rounded up to fit after the header in a power-of-2 - * slab object of up to 512 bytes (up to 29 slots on a 64-bit cpu). The slot - * array is not initialised. + * slab object of up to 512 bytes (up to 29 slots on a 64-bit cpu). The caller + * should be aware that the number of slots allocated may be more or less than + * the number requested. The slot array is not initialised. * * Return: The node pointer or NULL on allocation failure. */ @@ -56,16 +57,39 @@ struct bvecq *bvecq_alloc_one(size_t nr_slots, gfp_t gfp, bool for_writeback) const size_t max_slots = (max_size - sizeof(*bq)) / sizeof(bq->__bv[0]); size_t part = min(nr_slots, max_slots); size_t size = roundup_pow_of_two(struct_size(bq, __bv, part)); + bool from_pool = false; - bq = kmalloc(size, gfp & ~GFP_ZONEMASK); - if (!bq) - return bq; + gfp &= ~(GFP_ZONEMASK | __GFP_THISNODE); + + if (for_writeback) { + if (size != BVECQ_STD_SIZE) { + gfp_t gfp_temp = gfp; + gfp_temp |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; + gfp_temp &= ~(__GFP_DIRECT_RECLAIM | __GFP_IO); + bq = kmalloc(size, gfp_temp); + if (bq) + goto success; + } + + bq = mempool_alloc(&netfs_bvecq_pool, gfp); + if (!bq) + return bq; + from_pool = true; + size = BVECQ_STD_SIZE; + } else { + bq = kmalloc(size, gfp); + if (!bq) + return bq; + } + +success: *bq = (struct bvecq) { .ref = REFCOUNT_INIT(1), .bv = bq->__bv, .inline_bv = true, .max_slots = (size - sizeof(*bq)) / sizeof(bq->__bv[0]), + .from_pool = from_pool, }; netfs_stat(&netfs_n_bvecq); return bq; @@ -245,7 +269,10 @@ void bvecq_put(struct bvecq *bq) bvecq_free_slot(bq, slot); next = bq->next; netfs_stat_d(&netfs_n_bvecq); - kfree(bq); + if (bq->from_pool) + mempool_free(bq, &netfs_bvecq_pool); + else + kfree(bq); } } EXPORT_SYMBOL(bvecq_put); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index ef08f0096192..6d40a69e032f 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -43,6 +43,7 @@ extern struct list_head netfs_io_requests; extern spinlock_t netfs_proc_lock; extern mempool_t netfs_request_pool; extern mempool_t netfs_subrequest_pool; +extern mempool_t netfs_bvecq_pool; extern mempool_t netfs_folioq_pool; #ifdef CONFIG_PROC_FS diff --git a/fs/netfs/main.c b/fs/netfs/main.c index 927badf3989d..9f72e5054aff 100644 --- a/fs/netfs/main.c +++ b/fs/netfs/main.c @@ -28,6 +28,7 @@ static struct kmem_cache *netfs_request_slab; static struct kmem_cache *netfs_subrequest_slab; mempool_t netfs_request_pool; mempool_t netfs_subrequest_pool; +mempool_t netfs_bvecq_pool; mempool_t netfs_folioq_pool; #ifdef CONFIG_PROC_FS @@ -112,6 +113,9 @@ static int __init netfs_init(void) if (mempool_init_kmalloc_pool(&netfs_folioq_pool, 100, sizeof(struct folio_queue)) < 0) goto error_folioq_pool; + if (mempool_init_kmalloc_pool(&netfs_bvecq_pool, 100, BVECQ_STD_SIZE) < 0) + goto error_bvecq_pool; + netfs_request_slab = kmem_cache_create("netfs_request", sizeof(struct netfs_io_request), 0, SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, @@ -164,6 +168,8 @@ static int __init netfs_init(void) error_reqpool: kmem_cache_destroy(netfs_request_slab); error_req: + mempool_exit(&netfs_bvecq_pool); +error_bvecq_pool: mempool_exit(&netfs_folioq_pool); error_folioq_pool: return ret; @@ -178,6 +184,7 @@ static void __exit netfs_exit(void) kmem_cache_destroy(netfs_subrequest_slab); mempool_exit(&netfs_request_pool); kmem_cache_destroy(netfs_request_slab); + mempool_exit(&netfs_bvecq_pool); mempool_exit(&netfs_folioq_pool); } module_exit(netfs_exit); diff --git a/include/linux/bvecq.h b/include/linux/bvecq.h index 22fc7995f3ee..8adfdd43b865 100644 --- a/include/linux/bvecq.h +++ b/include/linux/bvecq.h @@ -43,15 +43,18 @@ struct bvecq { u16 max_slots; /* Number of elements allocated in bv[] */ enum bvecq_mem mem_type:3; /* What sort of memory and how to free it */ bool inline_bv:1; /* T if __bv[] is being used */ + bool from_pool:1; /* T if bvecq from mempool */ struct bio_vec *bv; /* Pointer to array of page fragments */ struct bio_vec __bv[]; /* Default array (if ->inline_bv) */ }; #if BITS_PER_LONG == 64 /* Number of slots in __bv[] for a bvecq in a 512-byte kmalloc block. */ +#define BVECQ_STD_SIZE 512 #define BVECQ_STD_SLOTS 29 /* 2 words/slot; 32 slots; bvecq is 6 words (3 slots) */ #elif BITS_PER_LONG == 32 /* Number of slots in __bv[] for a bvecq in a 256-byte kmalloc block. */ +#define BVECQ_STD_SIZE 256 #define BVECQ_STD_SLOTS 18 /* 3 words/slot; 21 slots; bvecq is 9 words (3 slots) */ #else #error BVECQ_STD_SLOTS undetermined
