Re: [for-4.16 PATCH v6 2/4] block: properly protect the 'queue' kobj in blk_unregister_queue
On Fri, Jan 12, 2018 at 11:03:52AM -0500, Mike Snitzer wrote:
> The original commit e9a823fb34a8b (block: fix warning when I/O elevator
> is changed as request_queue is being removed) is pretty conflated.
> "conflated" because the resource being protected by q->sysfs_lock isn't
> the queue_flags (it is the 'queue' kobj).
>
> q->sysfs_lock serializes __elevator_change() (via elv_iosched_store)
> from racing with blk_unregister_queue():
> 1) By holding q->sysfs_lock first, __elevator_change() can complete
> before a racing blk_unregister_queue().
> 2) Conversely, __elevator_change() is testing for QUEUE_FLAG_REGISTERED
> in case elv_iosched_store() loses the race with blk_unregister_queue(),
> it needs a way to know the 'queue' kobj isn't there.
>
> Expand the scope of blk_unregister_queue()'s q->sysfs_lock use so it is
> held until after the 'queue' kobj is removed.
>
> To do so blk_mq_unregister_dev() must not also take q->sysfs_lock. So
> rename __blk_mq_unregister_dev() to blk_mq_unregister_dev().
>
> Also, blk_unregister_queue() should use q->queue_lock to protect against
> any concurrent writes to q->queue_flags -- even though chances are the
> queue is being cleaned up so no concurrent writes are likely.
>
> Fixes: e9a823fb34a8b ("block: fix warning when I/O elevator is changed as
> request_queue is being removed")
> Signed-off-by: Mike Snitzer
> ---
> block/blk-mq-sysfs.c | 9 +
> block/blk-sysfs.c| 13 ++---
> 2 files changed, 11 insertions(+), 11 deletions(-)
>
> v6: blk_mq_unregister_dev now requires q->sysfs_lock be held, Ming: I am
> not seeing any lockdep complaints with this. I've tested bio-based,
> blk-mq and old .request_fn request-based.
>
> diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
> index 79969c3c234f..a54b4b070f1c 100644
> --- a/block/blk-mq-sysfs.c
> +++ b/block/blk-mq-sysfs.c
> @@ -248,7 +248,7 @@ static int blk_mq_register_hctx(struct blk_mq_hw_ctx
> *hctx)
> return ret;
> }
>
> -static void __blk_mq_unregister_dev(struct device *dev, struct request_queue
> *q)
> +void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
> {
> struct blk_mq_hw_ctx *hctx;
> int i;
> @@ -265,13 +265,6 @@ static void __blk_mq_unregister_dev(struct device *dev,
> struct request_queue *q)
> q->mq_sysfs_init_done = false;
> }
>
> -void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
> -{
> - mutex_lock(>sysfs_lock);
> - __blk_mq_unregister_dev(dev, q);
> - mutex_unlock(>sysfs_lock);
> -}
> -
> void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
> {
> kobject_init(>kobj, _mq_hw_ktype);
> diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
> index 870484eaed1f..9272452ff456 100644
> --- a/block/blk-sysfs.c
> +++ b/block/blk-sysfs.c
> @@ -929,12 +929,17 @@ void blk_unregister_queue(struct gendisk *disk)
> if (WARN_ON(!q))
> return;
>
> + /*
> + * Protect against the 'queue' kobj being accessed
> + * while/after it is removed.
> + */
> mutex_lock(>sysfs_lock);
> - queue_flag_clear_unlocked(QUEUE_FLAG_REGISTERED, q);
> - mutex_unlock(>sysfs_lock);
>
> - wbt_exit(q);
> + spin_lock_irq(q->queue_lock);
> + queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
> + spin_unlock_irq(q->queue_lock);
>
> + wbt_exit(q);
>
> if (q->mq_ops)
> blk_mq_unregister_dev(disk_to_dev(disk), q);
> @@ -946,4 +951,6 @@ void blk_unregister_queue(struct gendisk *disk)
> kobject_del(>kobj);
> blk_trace_remove_sysfs(disk_to_dev(disk));
> kobject_put(_to_dev(disk)->kobj);
> +
> + mutex_unlock(>sysfs_lock);
> }
Reviewed-by: Ming Lei
--
Ming
[for-4.16 PATCH v6 2/4] block: properly protect the 'queue' kobj in blk_unregister_queue
The original commit e9a823fb34a8b (block: fix warning when I/O elevator
is changed as request_queue is being removed) is pretty conflated.
"conflated" because the resource being protected by q->sysfs_lock isn't
the queue_flags (it is the 'queue' kobj).
q->sysfs_lock serializes __elevator_change() (via elv_iosched_store)
from racing with blk_unregister_queue():
1) By holding q->sysfs_lock first, __elevator_change() can complete
before a racing blk_unregister_queue().
2) Conversely, __elevator_change() is testing for QUEUE_FLAG_REGISTERED
in case elv_iosched_store() loses the race with blk_unregister_queue(),
it needs a way to know the 'queue' kobj isn't there.
Expand the scope of blk_unregister_queue()'s q->sysfs_lock use so it is
held until after the 'queue' kobj is removed.
To do so blk_mq_unregister_dev() must not also take q->sysfs_lock. So
rename __blk_mq_unregister_dev() to blk_mq_unregister_dev().
Also, blk_unregister_queue() should use q->queue_lock to protect against
any concurrent writes to q->queue_flags -- even though chances are the
queue is being cleaned up so no concurrent writes are likely.
Fixes: e9a823fb34a8b ("block: fix warning when I/O elevator is changed as
request_queue is being removed")
Signed-off-by: Mike Snitzer
---
block/blk-mq-sysfs.c | 9 +
block/blk-sysfs.c| 13 ++---
2 files changed, 11 insertions(+), 11 deletions(-)
v6: blk_mq_unregister_dev now requires q->sysfs_lock be held, Ming: I am
not seeing any lockdep complaints with this. I've tested bio-based,
blk-mq and old .request_fn request-based.
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 79969c3c234f..a54b4b070f1c 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -248,7 +248,7 @@ static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
return ret;
}
-static void __blk_mq_unregister_dev(struct device *dev, struct request_queue
*q)
+void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
int i;
@@ -265,13 +265,6 @@ static void __blk_mq_unregister_dev(struct device *dev,
struct request_queue *q)
q->mq_sysfs_init_done = false;
}
-void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
-{
- mutex_lock(>sysfs_lock);
- __blk_mq_unregister_dev(dev, q);
- mutex_unlock(>sysfs_lock);
-}
-
void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
kobject_init(>kobj, _mq_hw_ktype);
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 870484eaed1f..9272452ff456 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -929,12 +929,17 @@ void blk_unregister_queue(struct gendisk *disk)
if (WARN_ON(!q))
return;
+ /*
+* Protect against the 'queue' kobj being accessed
+* while/after it is removed.
+*/
mutex_lock(>sysfs_lock);
- queue_flag_clear_unlocked(QUEUE_FLAG_REGISTERED, q);
- mutex_unlock(>sysfs_lock);
- wbt_exit(q);
+ spin_lock_irq(q->queue_lock);
+ queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
+ spin_unlock_irq(q->queue_lock);
+ wbt_exit(q);
if (q->mq_ops)
blk_mq_unregister_dev(disk_to_dev(disk), q);
@@ -946,4 +951,6 @@ void blk_unregister_queue(struct gendisk *disk)
kobject_del(>kobj);
blk_trace_remove_sysfs(disk_to_dev(disk));
kobject_put(_to_dev(disk)->kobj);
+
+ mutex_unlock(>sysfs_lock);
}
--
2.15.0
