From: Yu Kuai <[email protected]> After radix tree preloading is gone, callers no longer need to allocate a blkg before entering blkg_create(). Move allocation into blkg_create() and pass the desired GFP mask instead.
Use GFP_NOIO for runtime and config blkg creation so slow paths can sleep without recursing into IO reclaim, keep GFP_KERNEL for root blkg setup, and use GFP_ATOMIC when nowait bio association creates a missing blkg after a successful q->blkcg_mutex trylock. Signed-off-by: Yu Kuai <[email protected]> --- block/blk-cgroup.c | 89 ++++++++++------------------------------------ 1 file changed, 18 insertions(+), 71 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index b99ab8d67798..ddc9073d7ab9 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -371,14 +371,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, return NULL; } -/* - * If @new_blkg is %NULL, this function tries to allocate a new one as - * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return. - */ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, - struct blkcg_gq *new_blkg) + gfp_t gfp_mask) { - struct blkcg_gq *blkg; + struct blkcg_gq *blkg = NULL; int i, ret; lockdep_assert_held(&disk->queue->blkcg_mutex); @@ -389,15 +385,11 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, goto err_free_blkg; } - /* allocate */ - if (!new_blkg) { - new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT); - if (unlikely(!new_blkg)) { - ret = -ENOMEM; - goto err_free_blkg; - } + blkg = blkg_alloc(blkcg, disk, gfp_mask); + if (unlikely(!blkg)) { + ret = -ENOMEM; + goto err_free_blkg; } - blkg = new_blkg; /* link parent */ if (blkcg_parent(blkcg)) { @@ -447,8 +439,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, return ERR_PTR(ret); err_free_blkg: - if (new_blkg) - blkg_free(new_blkg); + if (blkg) + blkg_free(blkg); return ERR_PTR(ret); } @@ -505,7 +497,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, } rcu_read_unlock(); - blkg = blkg_create(pos, disk, NULL); + blkg = blkg_create(pos, disk, GFP_NOIO); if (IS_ERR(blkg)) { blkg = ret_blkg; break; @@ -858,7 +850,6 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, while (true) { struct blkcg *pos = blkcg; struct blkcg *parent; - struct blkcg_gq *new_blkg; parent = blkcg_parent(blkcg); rcu_read_lock(); @@ -868,14 +859,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, } rcu_read_unlock(); - new_blkg = blkg_alloc(pos, disk, GFP_NOIO); - if (unlikely(!new_blkg)) { - ret = -ENOMEM; - goto fail_unlock; - } - if (!blkcg_policy_enabled(q, pol)) { - blkg_free(new_blkg); ret = -EOPNOTSUPP; goto fail_unlock; } @@ -883,10 +867,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); blkg = blkg_lookup(pos, q); rcu_read_unlock(); - if (blkg) { - blkg_free(new_blkg); - } else { - blkg = blkg_create(pos, disk, new_blkg); + if (!blkg) { + blkg = blkg_create(pos, disk, GFP_NOIO); if (IS_ERR(blkg)) { ret = PTR_ERR(blkg); goto fail_unlock; @@ -1436,7 +1418,7 @@ void blkg_init_queue(struct request_queue *q) int blkcg_init_disk(struct gendisk *disk) { struct request_queue *q = disk->queue; - struct blkcg_gq *new_blkg, *blkg; + struct blkcg_gq *blkg; /* * If the queue is shared across disk rebind (e.g., SCSI), the @@ -1450,13 +1432,9 @@ int blkcg_init_disk(struct gendisk *disk) */ wait_var_event(&q->root_blkg, !READ_ONCE(q->root_blkg)); - new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); - if (!new_blkg) - return -ENOMEM; - /* Make sure the root blkg exists. */ mutex_lock(&q->blkcg_mutex); - blkg = blkg_create(&blkcg_root, disk, new_blkg); + blkg = blkg_create(&blkcg_root, disk, GFP_KERNEL); if (IS_ERR(blkg)) goto err_unlock; q->root_blkg = blkg; @@ -1559,8 +1537,7 @@ static void blkg_free_policy_data(struct blkcg_gq *blkg, int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) { struct request_queue *q = disk->queue; - struct blkg_policy_data *pd_prealloc = NULL; - struct blkcg_gq *blkg, *pinned_blkg = NULL; + struct blkcg_gq *blkg; unsigned int memflags; int ret; @@ -1578,7 +1555,6 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); -retry: mutex_lock(&q->blkcg_mutex); /* blkg_list is pushed at the head, reverse walk to initialize parents first */ @@ -1590,34 +1566,9 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (hlist_unhashed(&blkg->blkcg_node)) continue; - /* If prealloc matches, use it; otherwise try GFP_NOWAIT. */ - if (blkg == pinned_blkg) { - pd = pd_prealloc; - pd_prealloc = NULL; - } else { - pd = pol->pd_alloc_fn(disk, blkg->blkcg, GFP_NOWAIT); - } - - if (!pd) { - /* - * GFP_NOWAIT failed. Free the existing one and - * prealloc for @blkg w/ GFP_KERNEL. - */ - if (pinned_blkg) - blkg_put(pinned_blkg); - blkg_get(blkg); - pinned_blkg = blkg; - - mutex_unlock(&q->blkcg_mutex); - - if (pd_prealloc) - pol->pd_free_fn(pd_prealloc); - pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg, - GFP_KERNEL); - if (pd_prealloc) - goto retry; + pd = pol->pd_alloc_fn(disk, blkg->blkcg, GFP_NOIO); + if (!pd) goto enomem; - } spin_lock_irq(&blkg->blkcg->lock); @@ -1642,15 +1593,10 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); - if (pinned_blkg) - blkg_put(pinned_blkg); - if (pd_prealloc) - pol->pd_free_fn(pd_prealloc); return ret; enomem: /* alloc failed, take down everything */ - mutex_lock(&q->blkcg_mutex); list_for_each_entry(blkg, &q->blkg_list, q_node) blkg_free_policy_data(blkg, pol); ret = -ENOMEM; @@ -2080,7 +2026,8 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, if (!preemptible() || !mutex_trylock(&q->blkcg_mutex)) return NULL; - blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk); + blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk, + GFP_ATOMIC); if (blkg) blkg = blkg_lookup_tryget(blkg); mutex_unlock(&q->blkcg_mutex); -- 2.51.0
