On Thu 30-07-26 02:57:52, Tal Zussman wrote: > Some bio completion handlers need to run from preemptible task context, > but bio_endio() may be called from IRQ context (e.g., buffer_head > writeback). Callers need a way to ensure their callback eventually runs > from a sleepable context. Add infrastructure for that, in two forms: > > 1. BIO_COMPLETE_IN_TASK, a bio flag the submitter sets when it knows > in advance that its callback needs task context (e.g., dropbehind > writeback). bio_endio() sees the flag and offloads completion to a > worker automatically. > > 2. bio_complete_in_task(), a helper that completion callbacks can > invoke from within bi_end_io() when the deferral decision is > dynamic (e.g., fserror reporting). > > Both share a per-CPU list drained by a work item on a WQ_PERCPU > workqueue. Producers push the bio onto the local CPU's list and schedule > the work item, which then dispatches each bio's bi_end_io() from task > context. > > Both methods are gated on bio_in_atomic(), which returns true in any > context where a sleeping bi_end_io() is unsafe, including > non-preemptible task context. > > Two CPU hotplug callbacks are used to drain remaining bios from the > departing CPU's batch, while maintaining the per-CPU behavior. The > CPUHP_AP_ONLINE_DYN callback disables the per-CPU work item while the > CPU is still online, preventing it from running on an unbound worker > later. CPUHP_BP_PREPARE_DYN then drains any bios added between disabling > the work item and CPU offline. > > Link: https://lore.kernel.org/all/[email protected]/ > Suggested-by: Matthew Wilcox <[email protected]> > Suggested-by: Christoph Hellwig <[email protected]> > Signed-off-by: Tal Zussman <[email protected]>
Looks good! Feel free to add: Reviewed-by: Jan Kara <[email protected]> Honza > --- > block/bio.c | 132 > +++++++++++++++++++++++++++++++++++++++++++++- > include/linux/bio.h | 24 +++++++++ > include/linux/blk_types.h | 1 + > 3 files changed, 156 insertions(+), 1 deletion(-) > > diff --git a/block/bio.c b/block/bio.c > index 6a2f6fc3413e..a9610950325b 100644 > --- a/block/bio.c > +++ b/block/bio.c > @@ -1741,6 +1741,61 @@ void bio_check_pages_dirty(struct bio *bio) > schedule_work(&bio_dirty_work); > } > > +/* > + * Infrastructure for deferring bio completions to task-context via a per-CPU > + * workqueue. Triggered either by the BIO_COMPLETE_IN_TASK bio flag (static > + * decision at submit time) or by calling bio_complete_in_task() from > + * bi_end_io() (dynamic decision at completion time). > + */ > + > +struct bio_complete_batch { > + struct bio_list list; > + struct work_struct work; > + int cpu; > +}; > + > +static DEFINE_PER_CPU(struct bio_complete_batch, bio_complete_batch); > +static struct workqueue_struct *bio_complete_wq; > + > +static void bio_complete_work_fn(struct work_struct *w) > +{ > + struct bio_complete_batch *batch = > + container_of(w, struct bio_complete_batch, work); > + > + while (1) { > + struct bio_list list; > + struct bio *bio; > + > + local_irq_disable(); > + list = batch->list; > + bio_list_init(&batch->list); > + local_irq_enable(); > + > + if (bio_list_empty(&list)) > + break; > + > + while ((bio = bio_list_pop(&list))) > + bio->bi_end_io(bio); > + } > +} > + > +void __bio_complete_in_task(struct bio *bio) > +{ > + struct bio_complete_batch *batch; > + unsigned long flags; > + bool was_empty; > + > + local_irq_save(flags); > + batch = this_cpu_ptr(&bio_complete_batch); > + was_empty = bio_list_empty(&batch->list); > + bio_list_add(&batch->list, bio); > + local_irq_restore(flags); > + > + if (was_empty) > + queue_work_on(batch->cpu, bio_complete_wq, &batch->work); > +} > +EXPORT_SYMBOL_GPL(__bio_complete_in_task); > + > static inline bool bio_remaining_done(struct bio *bio) > { > /* > @@ -1815,7 +1870,9 @@ void bio_endio(struct bio *bio) > } > #endif > > - if (bio->bi_end_io) > + if (bio_flagged(bio, BIO_COMPLETE_IN_TASK) && bio_in_atomic()) > + __bio_complete_in_task(bio); > + else if (bio->bi_end_io) > bio->bi_end_io(bio); > } > EXPORT_SYMBOL(bio_endio); > @@ -2001,6 +2058,55 @@ int bioset_init(struct bio_set *bs, > } > EXPORT_SYMBOL(bioset_init); > > +static int bio_complete_batch_cpu_online(unsigned int cpu) > +{ > + struct bio_complete_batch *batch = &per_cpu(bio_complete_batch, cpu); > + > + enable_work(&batch->work); > + if (!bio_list_empty(&batch->list)) > + queue_work_on(cpu, bio_complete_wq, &batch->work); > + return 0; > +} > + > +/* > + * Disable this CPU's work item so that it cannot run on an unbound worker > + * after the CPU is offlined. > + */ > +static int bio_complete_batch_cpu_down_prep(unsigned int cpu) > +{ > + disable_work_sync(&per_cpu(bio_complete_batch, cpu).work); > + return 0; > +} > + > +/* > + * Drain a dead CPU's deferred bio completions. The CPU is dead and the > worker > + * is canceled so no locking is needed. > + */ > +static int bio_complete_batch_cpu_dead(unsigned int cpu) > +{ > + struct bio_complete_batch *batch = > + per_cpu_ptr(&bio_complete_batch, cpu); > + struct bio *bio; > + > + while ((bio = bio_list_pop(&batch->list))) > + bio->bi_end_io(bio); > + > + return 0; > +} > + > +static void __init bio_complete_batch_init(int cpu) > +{ > + struct bio_complete_batch *batch = > + per_cpu_ptr(&bio_complete_batch, cpu); > + > + bio_list_init(&batch->list); > + INIT_WORK(&batch->work, bio_complete_work_fn); > + batch->cpu = cpu; > + > + if (!cpu_online(cpu)) > + disable_work_sync(&batch->work); > +} > + > static int __init init_bio(void) > { > int i; > @@ -2015,6 +2121,30 @@ static int __init init_bio(void) > SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); > } > > + for_each_possible_cpu(i) > + bio_complete_batch_init(i); > + > + bio_complete_wq = alloc_workqueue("bio_complete", > + WQ_MEM_RECLAIM | WQ_PERCPU, 0); > + if (!bio_complete_wq) > + panic("bio: can't allocate bio_complete workqueue\n"); > + > + /* > + * bio task-context completion draining on hot-unplugged CPUs: > + * > + * 1. Stop the per-CPU work item while the CPU is still online, so > + * that it cannot run on an unbound worker later. > + * 2. Drain leftover bios added between worker disabling and CPU > + * offlining. > + */ > + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, > + "block/bio:complete:online", > + bio_complete_batch_cpu_online, > + bio_complete_batch_cpu_down_prep); > + cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, > + "block/bio:complete:dead", > + NULL, bio_complete_batch_cpu_dead); > + > cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, > bio_cpu_dead); > > diff --git a/include/linux/bio.h b/include/linux/bio.h > index 62a983758e09..a8091a3e9d87 100644 > --- a/include/linux/bio.h > +++ b/include/linux/bio.h > @@ -383,6 +383,30 @@ static inline bool bio_in_atomic(void) > return !preemptible(); > } > > +void __bio_complete_in_task(struct bio *bio); > + > +/** > + * bio_complete_in_task - ensure a bio is completed in preemptible task > context > + * @bio: bio to complete > + * > + * If called from non-task context, offload the bio completion to a worker > + * thread and return %true. Else return %false and do nothing. > + * > + * Uses BIO_COMPLETE_IN_TASK as a sentinel: if set, the bio was already > + * deferred and we are running in the worker — return %false so the > + * callback proceeds instead of re-deferring. > + */ > +static inline bool bio_complete_in_task(struct bio *bio) > +{ > + if (bio_flagged(bio, BIO_COMPLETE_IN_TASK)) > + return false; > + if (!bio_in_atomic()) > + return false; > + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); > + __bio_complete_in_task(bio); > + return true; > +} > + > extern void bio_endio(struct bio *); > > /** > diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h > index 8808ee76e73c..d49d97a050d0 100644 > --- a/include/linux/blk_types.h > +++ b/include/linux/blk_types.h > @@ -322,6 +322,7 @@ enum { > BIO_REMAPPED, > BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */ > BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */ > + BIO_COMPLETE_IN_TASK, /* complete bi_end_io() in task context */ > BIO_FLAG_LAST > }; > > > -- > 2.39.5 > -- Jan Kara <[email protected]> SUSE Labs, CR
