queue_work() is pretty costly under high load. It takes pool->lock; unbound pools do not track running workers (WORKER_UNBOUND is part of WORKER_NOT_RUNNING), so an idle kworker is woken (idle exit + IPI + context switch) even if a running worker would pick the work up right away. Every qio has its own work, so this is paid for every I/O. With several submitters this ends up in several context switches per I/O and in contention on pool->lock.
Batch requests with percpu llists. Additionally track if the worker is busy and queue_work() only if it is not. The worker processes one batch per run and requeues itself if more qios came meanwhile, so flush_workqueue(), e.g. on backward merge completion with I/O still running, does not wait for a worker which is refilled all the time. The fast path is now serial per submitting CPU: qios queued from one CPU are processed one by one by its work, while qios from different CPUs are still processed in parallel. This is fine, since a fast path qio is only mapped and submitted as direct AIO: O_DIRECT is enforced for every delta. The exception is a partial discard, which punches the hole synchronously in the worker, so the qios queued after it on that CPU wait. Most of them would wait anyway: on ext4 and xfs punch hole takes the inode lock exclusively, and direct I/O submission to the same image sleeps on it. https://virtuozzo.atlassian.net/browse/VSTOR-95494 Feature: dm-qcow2: block device over QCOW2 files driver Signed-off-by: Andrey Zhadchenko <[email protected]> --- v2: moved before "declare DM_TARGET_NOWAIT"; process one batch per run and requeue, so flush_workqueue() does not wait for a work which is refilled all the time; commit message: fixed the claim about re-queued qios, describes that the fast path is serial per submitting CPU and that partial discards punch the hole in the worker, measurements moved to the thread drivers/md/dm-qcow2-map.c | 100 +++++++++++++++++++++++++++++++---- drivers/md/dm-qcow2-target.c | 10 +++- drivers/md/dm-qcow2.h | 6 ++- 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/drivers/md/dm-qcow2-map.c b/drivers/md/dm-qcow2-map.c index 7a8deba4b0a83..a87a396af56d0 100644 --- a/drivers/md/dm-qcow2-map.c +++ b/drivers/md/dm-qcow2-map.c @@ -87,6 +87,7 @@ static void process_read_qio(struct qcow2 *qcow2, struct qio *qio, struct qcow2_map *map); static void init_qrq_and_embedded_qio(struct qcow2_target *tgt, struct request *rq, struct qcow2_rq *qrq, struct qio *qio); +static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio); static loff_t bytes_off_in_cluster(struct qcow2 *qcow2, struct qio *qio) { @@ -362,8 +363,6 @@ struct qio *qcow2_alloc_qio(mempool_t *pool, bool zero) } ALLOW_ERROR_INJECTION(qcow2_alloc_qio, NULL); -static void do_qio_work(struct work_struct *ws); - void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2) { qio->qcow2 = qcow2; @@ -376,7 +375,6 @@ void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2) qio->ref_index = REF_INDEX_INVALID; qio->state = QIO_STATE_START; atomic_set(&qio->remaining, 1); - INIT_WORK(&qio->worker, do_qio_work); /* * Initially set into BLK_STS_OK, while aio complete, @@ -4511,7 +4509,7 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio) if (blk_rq_bytes(rq)) { queue_list_id = QLIST_EMBEDDED; - worker = &qio->worker; + worker = NULL; } else { WARN_ON_ONCE(qio->bi_op != REQ_OP_FLUSH); queue_list_id = QLIST_FLUSH; @@ -4534,8 +4532,13 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio) } spin_unlock_irqrestore(&qcow2->deferred_lock, flags); - if (queue) + if (!queue) + return; + + if (worker) queue_work(tgt->wq, worker); + else + qcow2_fast_submit(tgt, qio); } void qcow2_submit_embedded_qios(struct qcow2_target *tgt, struct list_head *list) @@ -4634,15 +4637,11 @@ static void qio_try_fast(struct qio *qio, struct list_head *deferred_qios) list_add_tail(&qio->link, deferred_qios); } -static void do_qio_work(struct work_struct *ws) +static void do_qio_work(struct qio *qio) { - struct qio *qio = container_of(ws, struct qio, worker); - struct qio *i, *next; struct qcow2 *qcow2 = qio->qcow2; struct qcow2_target *tgt = qcow2->tgt; - unsigned int pflags = current->flags; - - current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO; + struct qio *i, *next; switch (qio->state) { case QIO_STATE_START: { @@ -4668,8 +4667,87 @@ static void do_qio_work(struct work_struct *ws) qio_endio(qio); break; } +} + +struct qcow2_fast_cpu { + struct llist_head qios; + struct work_struct work; + struct qcow2_target *tgt; + atomic_t busy; +}; + +static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio) +{ + struct qcow2_fast_cpu *fc = get_cpu_ptr(tgt->fast); + + llist_add(&qio->fast_link, &fc->qios); + /* Pairs with smp_mb() in qcow2_fast_work() */ + smp_mb(); + if (!atomic_read(&fc->busy) && atomic_cmpxchg(&fc->busy, 0, 1) == 0) + queue_work(tgt->wq, &fc->work); + put_cpu_ptr(tgt->fast); +} + +static void qcow2_fast_work(struct work_struct *work) +{ + struct qcow2_fast_cpu *fc = container_of(work, struct qcow2_fast_cpu, work); + unsigned int pflags = current->flags; + struct llist_node *node; + struct qio *qio, *next; + + current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO; + + node = llist_del_all(&fc->qios); + node = llist_reverse_order(node); + llist_for_each_entry_safe(qio, next, node, fast_link) + do_qio_work(qio); current_restore_flags(pflags, PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO); + + if (llist_empty(&fc->qios)) { + atomic_set(&fc->busy, 0); + /* just to be sure we look at the list after busy is posted */ + smp_mb(); + if (llist_empty(&fc->qios)) + return; + if (atomic_cmpxchg(&fc->busy, 0, 1) != 0) + return; + } + queue_work(fc->tgt->wq, &fc->work); +} + +void qcow2_fast_exit(struct qcow2_target *tgt) +{ + struct qcow2_fast_cpu *fc; + int cpu; + + if (!tgt->fast) + return; + for_each_possible_cpu(cpu) { + fc = per_cpu_ptr(tgt->fast, cpu); + WARN_ON_ONCE(!llist_empty(&fc->qios)); + } + free_percpu(tgt->fast); + tgt->fast = NULL; +} + +int qcow2_fast_init(struct qcow2_target *tgt) +{ + struct qcow2_fast_cpu *fc; + int cpu; + + tgt->fast = alloc_percpu(struct qcow2_fast_cpu); + if (!tgt->fast) + return -ENOMEM; + for_each_possible_cpu(cpu) { + fc = per_cpu_ptr(tgt->fast, cpu); + init_llist_head(&fc->qios); + INIT_WORK(&fc->work, qcow2_fast_work); + fc->tgt = tgt; + atomic_set(&fc->busy, 0); + } + + return 0; } struct qio_data_llseek_hole { diff --git a/drivers/md/dm-qcow2-target.c b/drivers/md/dm-qcow2-target.c index 8337efc5c6238..3e85de5c8b050 100644 --- a/drivers/md/dm-qcow2-target.c +++ b/drivers/md/dm-qcow2-target.c @@ -253,6 +253,7 @@ static void qcow2_tgt_destroy(struct qcow2_target *tgt) /* Now kill the queue */ destroy_workqueue(tgt->wq); } + qcow2_fast_exit(tgt); qcow2_merge_backward_cancel(tgt); mempool_destroy(tgt->qio_pool); @@ -477,6 +478,11 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti) goto out_pool; } + if (qcow2_fast_init(tgt)) { + ti->error = "Can't alloc fast path slots"; + goto out_wq; + } + for (i = 0; i < 2; i++) { release = i ? inflight_ref_exit1 : inflight_ref_exit0; if (percpu_ref_init(&tgt->inflight_ref[i], release, @@ -484,7 +490,7 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti) if (i) percpu_ref_exit(&tgt->inflight_ref[0]); ti->error = "could not alloc percpu_ref"; - goto out_wq; + goto out_fast; } } @@ -504,6 +510,8 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti) qcow2_set_service_operations(ti, false); return tgt; +out_fast: + qcow2_fast_exit(tgt); out_wq: destroy_workqueue(tgt->wq); out_pool: diff --git a/drivers/md/dm-qcow2.h b/drivers/md/dm-qcow2.h index c044c630ca238..d6884eba2f9de 100644 --- a/drivers/md/dm-qcow2.h +++ b/drivers/md/dm-qcow2.h @@ -4,6 +4,7 @@ #include <linux/percpu-refcount.h> #include <linux/device-mapper.h> +#include <linux/llist.h> #include <linux/fs.h> #include <linux/eventfd.h> @@ -192,6 +193,7 @@ struct qcow2_target { */ struct qcow2 *top; struct workqueue_struct *wq; + struct qcow2_fast_cpu __percpu *fast; struct completion inflight_ref_comp; struct percpu_ref inflight_ref[2]; @@ -377,7 +379,7 @@ struct qio { struct qcow2 *qcow2; qcow2_endio_t endio_cb; void *endio_cb_data; - struct work_struct worker; + struct llist_node fast_link; int state; }; @@ -394,6 +396,8 @@ loff_t qcow2_llseek_hole(struct dm_target *ti, loff_t offset, int whence); void do_qcow2_work(struct work_struct *ws); void do_qcow2_fsync_work(struct work_struct *ws); +int qcow2_fast_init(struct qcow2_target *tgt); +void qcow2_fast_exit(struct qcow2_target *tgt); int qcow2_alloc_and_insert_md_page(struct qcow2 *qcow2, u64 index, struct md_page **md); struct md_page *qcow2_md_page_find_or_postpone(struct qcow2 *qcow2, -- 2.43.5 _______________________________________________ Devel mailing list [email protected] https://lists.openvz.org/mailman/listinfo/devel
