Protect sw-ctx busy bit manipulations via the sw queue lock. This allows to convert the atomic bit operations into slightly faster non-atomic operations. Document that blk_mq_run_hw_queues() tests these bits without holding the sw queue lock.
Signed-off-by: Bart Van Assche <[email protected]> Cc: Christoph Hellwig <[email protected]> Cc: Sagi Grimberg <[email protected]> --- block/blk-mq.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 13f5a6c..0dcafa6 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -66,8 +66,9 @@ static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, { struct blk_align_bitmap *bm = get_bm(hctx, ctx); + lockdep_assert_held(&ctx->lock); if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word)) - set_bit(CTX_TO_BIT(hctx, ctx), &bm->word); + __set_bit(CTX_TO_BIT(hctx, ctx), &bm->word); } static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, @@ -75,7 +76,8 @@ static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, { struct blk_align_bitmap *bm = get_bm(hctx, ctx); - clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word); + lockdep_assert_held(&ctx->lock); + __clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word); } void blk_mq_freeze_queue_start(struct request_queue *q) @@ -768,8 +770,8 @@ static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) break; ctx = hctx->ctxs[bit + off]; - clear_bit(bit, &bm->word); spin_lock(&ctx->lock); + __clear_bit(bit, &bm->word); list_splice_tail_init(&ctx->rq_list, list); spin_unlock(&ctx->lock); @@ -940,6 +942,13 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) &hctx->run_work, 0); } +/** + * blk_mq_run_hw_queues - run all hardware queues + * + * Note: this function checks the SW and HW busy status without serialization + * against the functions that modify that status information. The caller is + * responsible for realizing that serialization. + */ void blk_mq_run_hw_queues(struct request_queue *q, bool async) { struct blk_mq_hw_ctx *hctx; @@ -1055,6 +1064,7 @@ static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, { struct blk_mq_ctx *ctx = rq->mq_ctx; + lockdep_assert_held(&ctx->lock); __blk_mq_insert_req_list(hctx, rq, at_head); blk_mq_hctx_mark_pending(hctx, ctx); } -- 2.9.3 -- To unsubscribe from this list: send the line "unsubscribe linux-block" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
