This patch moves discard related code from segment.c into discard.c since discard is independent feature, and it's better to maintain them in separated place.
There is no functionality change. Signed-off-by: Yangtao Li <[email protected]> --- fs/f2fs/Makefile | 2 +- fs/f2fs/discard.c | 1271 +++++++++++++++++++++++++++++++++++++++++++++ fs/f2fs/f2fs.h | 37 +- fs/f2fs/segment.c | 1268 +------------------------------------------- 4 files changed, 1317 insertions(+), 1261 deletions(-) create mode 100644 fs/f2fs/discard.c diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile index 8a7322d229e4..3d2874633db0 100644 --- a/fs/f2fs/Makefile +++ b/fs/f2fs/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_F2FS_FS) += f2fs.o f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o -f2fs-y += shrinker.o extent_cache.o sysfs.o +f2fs-y += shrinker.o extent_cache.o sysfs.o discard.o f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o diff --git a/fs/f2fs/discard.c b/fs/f2fs/discard.c new file mode 100644 index 000000000000..794f88fc729d --- /dev/null +++ b/fs/f2fs/discard.c @@ -0,0 +1,1271 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs discard support + * + * Copyright (c) 2022 Vivo Communication Technology Co.,Ltd. + * Author: Yangtao Li <[email protected]> + */ + +#include <linux/f2fs_fs.h> +#include <linux/freezer.h> +#include <linux/kthread.h> + +#include "f2fs.h" +#include "segment.h" +#include "node.h" +#include "iostat.h" +#include <trace/events/f2fs.h> + +static struct kmem_cache *discard_entry_slab; +static struct kmem_cache *discard_cmd_slab; + +static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc; + + f2fs_bug_on(sbi, !len); + + pend_list = &dcc->pend_list[plist_idx(len)]; + + dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); + INIT_LIST_HEAD(&dc->list); + dc->bdev = bdev; + dc->lstart = lstart; + dc->start = start; + dc->len = len; + dc->ref = 0; + dc->state = D_PREP; + dc->queued = 0; + dc->error = 0; + init_completion(&dc->wait); + list_add_tail(&dc->list, pend_list); + spin_lock_init(&dc->lock); + dc->bio_ref = 0; + atomic_inc(&dcc->discard_cmd_cnt); + dcc->undiscard_blks += len; + + return dc; +} + +static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len, + struct rb_node *parent, struct rb_node **p, + bool leftmost) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *dc; + + dc = __create_discard_cmd(sbi, bdev, lstart, start, len); + + rb_link_node(&dc->rb_node, parent, p); + rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); + + return dc; +} + +static void __detach_discard_cmd(struct discard_cmd_control *dcc, + struct discard_cmd *dc) +{ + if (dc->state == D_DONE) + atomic_sub(dc->queued, &dcc->queued_discard); + + list_del(&dc->list); + rb_erase_cached(&dc->rb_node, &dcc->root); + dcc->undiscard_blks -= dc->len; + + kmem_cache_free(discard_cmd_slab, dc); + + atomic_dec(&dcc->discard_cmd_cnt); +} + +static void __remove_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_cmd *dc) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + unsigned long flags; + + trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); + + spin_lock_irqsave(&dc->lock, flags); + if (dc->bio_ref) { + spin_unlock_irqrestore(&dc->lock, flags); + return; + } + spin_unlock_irqrestore(&dc->lock, flags); + + f2fs_bug_on(sbi, dc->ref); + + if (dc->error == -EOPNOTSUPP) + dc->error = 0; + + if (dc->error) + printk_ratelimited( + "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d", + KERN_INFO, sbi->sb->s_id, + dc->lstart, dc->start, dc->len, dc->error); + __detach_discard_cmd(dcc, dc); +} + +static void f2fs_submit_discard_endio(struct bio *bio) +{ + struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; + unsigned long flags; + + spin_lock_irqsave(&dc->lock, flags); + if (!dc->error) + dc->error = blk_status_to_errno(bio->bi_status); + dc->bio_ref--; + if (!dc->bio_ref && dc->state == D_SUBMIT) { + dc->state = D_DONE; + complete_all(&dc->wait); + } + spin_unlock_irqrestore(&dc->lock, flags); + bio_put(bio); +} + +static void __init_discard_policy(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + int discard_type, unsigned int granularity) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + /* common policy */ + dpolicy->type = discard_type; + dpolicy->sync = true; + dpolicy->ordered = false; + dpolicy->granularity = granularity; + + dpolicy->max_requests = dcc->max_discard_request; + dpolicy->io_aware_gran = MAX_PLIST_NUM; + dpolicy->timeout = false; + + if (discard_type == DPOLICY_BG) { + dpolicy->min_interval = dcc->min_discard_issue_time; + dpolicy->mid_interval = dcc->mid_discard_issue_time; + dpolicy->max_interval = dcc->max_discard_issue_time; + dpolicy->io_aware = true; + dpolicy->sync = false; + dpolicy->ordered = true; + if (utilization(sbi) > dcc->discard_urgent_util) { + dpolicy->granularity = MIN_DISCARD_GRANULARITY; + if (atomic_read(&dcc->discard_cmd_cnt)) + dpolicy->max_interval = + dcc->min_discard_issue_time; + } + } else if (discard_type == DPOLICY_FORCE) { + dpolicy->min_interval = dcc->min_discard_issue_time; + dpolicy->mid_interval = dcc->mid_discard_issue_time; + dpolicy->max_interval = dcc->max_discard_issue_time; + dpolicy->io_aware = false; + } else if (discard_type == DPOLICY_FSTRIM) { + dpolicy->io_aware = false; + } else if (discard_type == DPOLICY_UMOUNT) { + dpolicy->io_aware = false; + /* we need to issue all to keep CP_TRIMMED_FLAG */ + dpolicy->granularity = MIN_DISCARD_GRANULARITY; + dpolicy->timeout = true; + } +} + +static void __update_discard_tree_range(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len); +/* this function is copied from blkdev_issue_discard from block/blk-lib.c */ +static int __submit_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + struct discard_cmd *dc, + unsigned int *issued) +{ + struct block_device *bdev = dc->bdev; + unsigned int max_discard_blocks = + SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? + &(dcc->fstrim_list) : &(dcc->wait_list); + blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; + block_t lstart, start, len, total_len; + int err = 0; + + if (dc->state != D_PREP) + return 0; + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) + return 0; + + trace_f2fs_issue_discard(bdev, dc->start, dc->len); + + lstart = dc->lstart; + start = dc->start; + len = dc->len; + total_len = len; + + dc->len = 0; + + while (total_len && *issued < dpolicy->max_requests && !err) { + struct bio *bio = NULL; + unsigned long flags; + bool last = true; + + if (len > max_discard_blocks) { + len = max_discard_blocks; + last = false; + } + + (*issued)++; + if (*issued == dpolicy->max_requests) + last = true; + + dc->len += len; + + if (time_to_inject(sbi, FAULT_DISCARD)) { + f2fs_show_injection_info(sbi, FAULT_DISCARD); + err = -EIO; + } else { + err = __blkdev_issue_discard(bdev, + SECTOR_FROM_BLOCK(start), + SECTOR_FROM_BLOCK(len), + GFP_NOFS, &bio); + } + if (err) { + spin_lock_irqsave(&dc->lock, flags); + if (dc->state == D_PARTIAL) + dc->state = D_SUBMIT; + spin_unlock_irqrestore(&dc->lock, flags); + + break; + } + + f2fs_bug_on(sbi, !bio); + + /* + * should keep before submission to avoid D_DONE + * right away + */ + spin_lock_irqsave(&dc->lock, flags); + if (last) + dc->state = D_SUBMIT; + else + dc->state = D_PARTIAL; + dc->bio_ref++; + spin_unlock_irqrestore(&dc->lock, flags); + + atomic_inc(&dcc->queued_discard); + dc->queued++; + list_move_tail(&dc->list, wait_list); + + /* sanity check on discard range */ + __check_sit_bitmap(sbi, lstart, lstart + len); + + bio->bi_private = dc; + bio->bi_end_io = f2fs_submit_discard_endio; + bio->bi_opf |= flag; + submit_bio(bio); + + atomic_inc(&dcc->issued_discard); + + f2fs_update_iostat(sbi, NULL, FS_DISCARD, len * F2FS_BLKSIZE); + + lstart += len; + start += len; + total_len -= len; + len = total_len; + } + + if (!err && len) { + dcc->undiscard_blks -= len; + __update_discard_tree_range(sbi, bdev, lstart, start, len); + } + return err; +} + +static void __insert_discard_tree(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len, + struct rb_node **insert_p, + struct rb_node *insert_parent) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct rb_node **p; + struct rb_node *parent = NULL; + bool leftmost = true; + + if (insert_p && insert_parent) { + parent = insert_parent; + p = insert_p; + goto do_insert; + } + + p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, + lstart, &leftmost); +do_insert: + __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, + p, leftmost); +} + +static void __relocate_discard_cmd(struct discard_cmd_control *dcc, + struct discard_cmd *dc) +{ + list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); +} + +static void __punch_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_cmd *dc, block_t blkaddr) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_info di = dc->di; + bool modified = false; + + if (dc->state == D_DONE || dc->len == 1) { + __remove_discard_cmd(sbi, dc); + return; + } + + dcc->undiscard_blks -= di.len; + + if (blkaddr > di.lstart) { + dc->len = blkaddr - dc->lstart; + dcc->undiscard_blks += dc->len; + __relocate_discard_cmd(dcc, dc); + modified = true; + } + + if (blkaddr < di.lstart + di.len - 1) { + if (modified) { + __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, + di.start + blkaddr + 1 - di.lstart, + di.lstart + di.len - 1 - blkaddr, + NULL, NULL); + } else { + dc->lstart++; + dc->len--; + dc->start++; + dcc->undiscard_blks += dc->len; + __relocate_discard_cmd(dcc, dc); + } + } +} + +static void __update_discard_tree_range(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct discard_cmd *dc; + struct discard_info di = {0}; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + unsigned int max_discard_blocks = + SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + block_t end = lstart + len; + + dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, + NULL, lstart, + (struct rb_entry **)&prev_dc, + (struct rb_entry **)&next_dc, + &insert_p, &insert_parent, true, NULL); + if (dc) + prev_dc = dc; + + if (!prev_dc) { + di.lstart = lstart; + di.len = next_dc ? next_dc->lstart - lstart : len; + di.len = min(di.len, len); + di.start = start; + } + + while (1) { + struct rb_node *node; + bool merged = false; + struct discard_cmd *tdc = NULL; + + if (prev_dc) { + di.lstart = prev_dc->lstart + prev_dc->len; + if (di.lstart < lstart) + di.lstart = lstart; + if (di.lstart >= end) + break; + + if (!next_dc || next_dc->lstart > end) + di.len = end - di.lstart; + else + di.len = next_dc->lstart - di.lstart; + di.start = start + di.lstart - lstart; + } + + if (!di.len) + goto next; + + if (prev_dc && prev_dc->state == D_PREP && + prev_dc->bdev == bdev && + __is_discard_back_mergeable(&di, &prev_dc->di, + max_discard_blocks)) { + prev_dc->di.len += di.len; + dcc->undiscard_blks += di.len; + __relocate_discard_cmd(dcc, prev_dc); + di = prev_dc->di; + tdc = prev_dc; + merged = true; + } + + if (next_dc && next_dc->state == D_PREP && + next_dc->bdev == bdev && + __is_discard_front_mergeable(&di, &next_dc->di, + max_discard_blocks)) { + next_dc->di.lstart = di.lstart; + next_dc->di.len += di.len; + next_dc->di.start = di.start; + dcc->undiscard_blks += di.len; + __relocate_discard_cmd(dcc, next_dc); + if (tdc) + __remove_discard_cmd(sbi, tdc); + merged = true; + } + + if (!merged) { + __insert_discard_tree(sbi, bdev, di.lstart, di.start, + di.len, NULL, NULL); + } + next: + prev_dc = next_dc; + if (!prev_dc) + break; + + node = rb_next(&prev_dc->rb_node); + next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); + } +} + +static void __queue_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ + block_t lblkstart = blkstart; + + if (!f2fs_bdev_support_discard(bdev)) + return; + + trace_f2fs_queue_discard(bdev, blkstart, blklen); + + if (f2fs_is_multi_device(sbi)) { + int devi = f2fs_target_device_index(sbi, blkstart); + + blkstart -= FDEV(devi).start_blk; + } + mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); + __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); + mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); +} + +static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + struct discard_cmd *dc; + struct blk_plug plug; + unsigned int pos = dcc->next_pos; + unsigned int issued = 0; + bool io_interrupted = false; + + mutex_lock(&dcc->cmd_lock); + dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, + NULL, pos, + (struct rb_entry **)&prev_dc, + (struct rb_entry **)&next_dc, + &insert_p, &insert_parent, true, NULL); + if (!dc) + dc = next_dc; + + blk_start_plug(&plug); + + while (dc) { + struct rb_node *node; + int err = 0; + + if (dc->state != D_PREP) + goto next; + + if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { + io_interrupted = true; + break; + } + + dcc->next_pos = dc->lstart + dc->len; + err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); + + if (issued >= dpolicy->max_requests) + break; +next: + node = rb_next(&dc->rb_node); + if (err) + __remove_discard_cmd(sbi, dc); + dc = rb_entry_safe(node, struct discard_cmd, rb_node); + } + + blk_finish_plug(&plug); + + if (!dc) + dcc->next_pos = 0; + + mutex_unlock(&dcc->cmd_lock); + + if (!issued && io_interrupted) + issued = -1; + + return issued; +} + +static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy); + +static int __issue_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc, *tmp; + struct blk_plug plug; + int i, issued; + bool io_interrupted = false; + + if (dpolicy->timeout) + f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); + +retry: + issued = 0; + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + if (dpolicy->timeout && + f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) + break; + + if (i + 1 < dpolicy->granularity) + break; + + if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) + return __issue_discard_cmd_orderly(sbi, dpolicy); + + pend_list = &dcc->pend_list[i]; + + mutex_lock(&dcc->cmd_lock); + if (list_empty(pend_list)) + goto next; + if (unlikely(dcc->rbtree_check)) + f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, + &dcc->root, false)); + blk_start_plug(&plug); + list_for_each_entry_safe(dc, tmp, pend_list, list) { + f2fs_bug_on(sbi, dc->state != D_PREP); + + if (dpolicy->timeout && + f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) + break; + + if (dpolicy->io_aware && i < dpolicy->io_aware_gran && + !is_idle(sbi, DISCARD_TIME)) { + io_interrupted = true; + break; + } + + __submit_discard_cmd(sbi, dpolicy, dc, &issued); + + if (issued >= dpolicy->max_requests) + break; + } + blk_finish_plug(&plug); +next: + mutex_unlock(&dcc->cmd_lock); + + if (issued >= dpolicy->max_requests || io_interrupted) + break; + } + + if (dpolicy->type == DPOLICY_UMOUNT && issued) { + __wait_all_discard_cmd(sbi, dpolicy); + goto retry; + } + + if (!issued && io_interrupted) + issued = -1; + + return issued; +} + +static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc, *tmp; + int i; + bool dropped = false; + + mutex_lock(&dcc->cmd_lock); + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + pend_list = &dcc->pend_list[i]; + list_for_each_entry_safe(dc, tmp, pend_list, list) { + f2fs_bug_on(sbi, dc->state != D_PREP); + __remove_discard_cmd(sbi, dc); + dropped = true; + } + } + mutex_unlock(&dcc->cmd_lock); + + return dropped; +} + +void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) +{ + __drop_discard_cmd(sbi); +} + +static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, + struct discard_cmd *dc) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + unsigned int len = 0; + + wait_for_completion_io(&dc->wait); + mutex_lock(&dcc->cmd_lock); + f2fs_bug_on(sbi, dc->state != D_DONE); + dc->ref--; + if (!dc->ref) { + if (!dc->error) + len = dc->len; + __remove_discard_cmd(sbi, dc); + } + mutex_unlock(&dcc->cmd_lock); + + return len; +} + +static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + block_t start, block_t end) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? + &(dcc->fstrim_list) : &(dcc->wait_list); + struct discard_cmd *dc = NULL, *iter, *tmp; + unsigned int trimmed = 0; + +next: + dc = NULL; + + mutex_lock(&dcc->cmd_lock); + list_for_each_entry_safe(iter, tmp, wait_list, list) { + if (iter->lstart + iter->len <= start || end <= iter->lstart) + continue; + if (iter->len < dpolicy->granularity) + continue; + if (iter->state == D_DONE && !iter->ref) { + wait_for_completion_io(&iter->wait); + if (!iter->error) + trimmed += iter->len; + __remove_discard_cmd(sbi, iter); + } else { + iter->ref++; + dc = iter; + break; + } + } + mutex_unlock(&dcc->cmd_lock); + + if (dc) { + trimmed += __wait_one_discard_bio(sbi, dc); + goto next; + } + + return trimmed; +} + +static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy) +{ + struct discard_policy dp; + unsigned int discard_blks; + + if (dpolicy) + return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); + + /* wait all */ + __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); + discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); + __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); + discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); + + return discard_blks; +} + +/* This should be covered by global mutex, &sit_i->sentry_lock */ +void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *dc; + bool need_wait = false; + + mutex_lock(&dcc->cmd_lock); + dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root, + NULL, blkaddr); + if (dc) { + if (dc->state == D_PREP) { + __punch_discard_cmd(sbi, dc, blkaddr); + } else { + dc->ref++; + need_wait = true; + } + } + mutex_unlock(&dcc->cmd_lock); + + if (need_wait) + __wait_one_discard_bio(sbi, dc); +} + +void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + if (dcc && dcc->f2fs_issue_discard) { + struct task_struct *discard_thread = dcc->f2fs_issue_discard; + + dcc->f2fs_issue_discard = NULL; + kthread_stop(discard_thread); + } +} + +/* This comes from f2fs_put_super */ +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_policy dpolicy; + bool dropped; + + if (!atomic_read(&dcc->discard_cmd_cnt)) + return false; + + __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, + dcc->discard_granularity); + __issue_discard_cmd(sbi, &dpolicy); + dropped = __drop_discard_cmd(sbi); + + /* just to make sure there is no pending discard commands */ + __wait_all_discard_cmd(sbi, NULL); + + f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); + return dropped; +} + +static int issue_discard_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + wait_queue_head_t *q = &dcc->discard_wait_queue; + struct discard_policy dpolicy; + unsigned int wait_ms = dcc->min_discard_issue_time; + int issued; + + set_freezable(); + + do { + wait_event_interruptible_timeout(*q, + kthread_should_stop() || freezing(current) || + dcc->discard_wake, + msecs_to_jiffies(wait_ms)); + + if (sbi->gc_mode == GC_URGENT_HIGH || + !f2fs_available_free_memory(sbi, DISCARD_CACHE)) + __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); + else + __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, + dcc->discard_granularity); + + if (dcc->discard_wake) + dcc->discard_wake = 0; + + /* clean up pending candidates before going to sleep */ + if (atomic_read(&dcc->queued_discard)) + __wait_all_discard_cmd(sbi, NULL); + + if (try_to_freeze()) + continue; + if (f2fs_readonly(sbi->sb)) + continue; + if (kthread_should_stop()) + return 0; + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || + !atomic_read(&dcc->discard_cmd_cnt)) { + wait_ms = dpolicy.max_interval; + continue; + } + + sb_start_intwrite(sbi->sb); + + issued = __issue_discard_cmd(sbi, &dpolicy); + if (issued > 0) { + __wait_all_discard_cmd(sbi, &dpolicy); + wait_ms = dpolicy.min_interval; + } else if (issued == -1) { + wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); + if (!wait_ms) + wait_ms = dpolicy.mid_interval; + } else { + wait_ms = dpolicy.max_interval; + } + if (!atomic_read(&dcc->discard_cmd_cnt)) + wait_ms = dpolicy.max_interval; + + sb_end_intwrite(sbi->sb); + + } while (!kthread_should_stop()); + return 0; +} + +int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ +#ifdef CONFIG_BLK_DEV_ZONED + sector_t sector, nr_sects; + block_t lblkstart = blkstart; + int devi = 0; + + if (f2fs_is_multi_device(sbi)) { + devi = f2fs_target_device_index(sbi, blkstart); + if (blkstart < FDEV(devi).start_blk || + blkstart > FDEV(devi).end_blk) { + f2fs_err(sbi, "Invalid block %x", blkstart); + return -EIO; + } + blkstart -= FDEV(devi).start_blk; + } + + /* For sequential zones, reset the zone write pointer */ + if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { + sector = SECTOR_FROM_BLOCK(blkstart); + nr_sects = SECTOR_FROM_BLOCK(blklen); + + if (sector & (bdev_zone_sectors(bdev) - 1) || + nr_sects != bdev_zone_sectors(bdev)) { + f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", + devi, sbi->s_ndevs ? FDEV(devi).path : "", + blkstart, blklen); + return -EIO; + } + trace_f2fs_issue_reset_zone(bdev, blkstart); + return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, + sector, nr_sects, GFP_NOFS); + } + + /* For conventional zones, use regular discard if supported */ + __queue_discard_cmd(sbi, bdev, lblkstart, blklen); +#endif + return 0; +} + +static int __issue_discard_async(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) + return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); +#endif + __queue_discard_cmd(sbi, bdev, blkstart, blklen); + return 0; +} + +int f2fs_issue_discard(struct f2fs_sb_info *sbi, + block_t blkstart, block_t blklen) +{ + sector_t start = blkstart, len = 0; + struct block_device *bdev; + struct seg_entry *se; + unsigned int offset; + block_t i; + int err = 0; + + bdev = f2fs_target_device(sbi, blkstart, NULL); + + for (i = blkstart; i < blkstart + blklen; i++, len++) { + if (i != start) { + struct block_device *bdev2 = + f2fs_target_device(sbi, i, NULL); + + if (bdev2 != bdev) { + err = __issue_discard_async(sbi, bdev, + start, len); + if (err) + return err; + bdev = bdev2; + start = i; + len = 0; + } + } + + se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); + offset = GET_BLKOFF_FROM_SEG0(sbi, i); + + if (f2fs_block_unit_discard(sbi) && + !f2fs_test_and_set_bit(offset, se->discard_map)) + sbi->discard_blks--; + } + + if (len) + err = __issue_discard_async(sbi, bdev, start, len); + return err; +} + +bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, + bool check_only) +{ + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + int max_blocks = sbi->blocks_per_seg; + struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *discard_map = (unsigned long *)se->discard_map; + unsigned long *dmap = SIT_I(sbi)->tmp_map; + unsigned int start = 0, end = -1; + bool force = (cpc->reason & CP_DISCARD); + struct discard_entry *de = NULL; + struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; + int i; + + if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) || + !f2fs_block_unit_discard(sbi)) + return false; + + if (!force) { + if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || + SM_I(sbi)->dcc_info->nr_discards >= + SM_I(sbi)->dcc_info->max_discards) + return false; + } + + /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ + for (i = 0; i < entries; i++) + dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : + (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; + + while (force || SM_I(sbi)->dcc_info->nr_discards <= + SM_I(sbi)->dcc_info->max_discards) { + start = __find_rev_next_bit(dmap, max_blocks, end + 1); + if (start >= max_blocks) + break; + + end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); + if (force && start && end != max_blocks + && (end - start) < cpc->trim_minlen) + continue; + + if (check_only) + return true; + + if (!de) { + de = f2fs_kmem_cache_alloc(discard_entry_slab, + GFP_F2FS_ZERO, true, NULL); + de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); + list_add_tail(&de->list, head); + } + + for (i = start; i < end; i++) + __set_bit_le(i, (void *)de->discard_map); + + SM_I(sbi)->dcc_info->nr_discards += end - start; + } + return false; +} + +void release_discard_addr(struct discard_entry *entry) +{ + list_del(&entry->list); + kmem_cache_free(discard_entry_slab, entry); +} + +void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) +{ + struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); + struct discard_entry *entry, *this; + + /* drop caches */ + list_for_each_entry_safe(entry, this, head, list) + release_discard_addr(entry); +} + +int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + int err = 0; + + if (!f2fs_realtime_discard_enable(sbi)) + return 0; + + dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, + "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(dcc->f2fs_issue_discard)) { + err = PTR_ERR(dcc->f2fs_issue_discard); + dcc->f2fs_issue_discard = NULL; + } + + return err; +} + +int create_discard_cmd_control(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc; + int err = 0, i; + + if (SM_I(sbi)->dcc_info) { + dcc = SM_I(sbi)->dcc_info; + goto init_thread; + } + + dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; + dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; + if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) + dcc->discard_granularity = sbi->blocks_per_seg; + else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) + dcc->discard_granularity = BLKS_PER_SEC(sbi); + + INIT_LIST_HEAD(&dcc->entry_list); + for (i = 0; i < MAX_PLIST_NUM; i++) + INIT_LIST_HEAD(&dcc->pend_list[i]); + INIT_LIST_HEAD(&dcc->wait_list); + INIT_LIST_HEAD(&dcc->fstrim_list); + mutex_init(&dcc->cmd_lock); + atomic_set(&dcc->issued_discard, 0); + atomic_set(&dcc->queued_discard, 0); + atomic_set(&dcc->discard_cmd_cnt, 0); + dcc->nr_discards = 0; + dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; + dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; + dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; + dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; + dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; + dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; + dcc->undiscard_blks = 0; + dcc->next_pos = 0; + dcc->root = RB_ROOT_CACHED; + dcc->rbtree_check = false; + + init_waitqueue_head(&dcc->discard_wait_queue); + SM_I(sbi)->dcc_info = dcc; +init_thread: + err = f2fs_start_discard_thread(sbi); + if (err) { + kfree(dcc); + SM_I(sbi)->dcc_info = NULL; + } + + return err; +} + +void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + if (!dcc) + return; + + f2fs_stop_discard_thread(sbi); + + /* + * Recovery can cache discard commands, so in error path of + * fill_super(), it needs to give a chance to handle them. + */ + f2fs_issue_discard_timeout(sbi); + + kfree(dcc); + SM_I(sbi)->dcc_info = NULL; +} + +bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, + struct cp_control *cpc) +{ + __u64 trim_start = cpc->trim_start; + bool has_candidate = false; + + down_write(&SIT_I(sbi)->sentry_lock); + for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { + if (add_discard_addrs(sbi, cpc, true)) { + has_candidate = true; + break; + } + } + up_write(&SIT_I(sbi)->sentry_lock); + + cpc->trim_start = trim_start; + return has_candidate; +} + +static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + unsigned int start, unsigned int end) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + struct discard_cmd *dc; + struct blk_plug plug; + int issued; + unsigned int trimmed = 0; + +next: + issued = 0; + + mutex_lock(&dcc->cmd_lock); + if (unlikely(dcc->rbtree_check)) + f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, + &dcc->root, false)); + + dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, + NULL, start, + (struct rb_entry **)&prev_dc, + (struct rb_entry **)&next_dc, + &insert_p, &insert_parent, true, NULL); + if (!dc) + dc = next_dc; + + blk_start_plug(&plug); + + while (dc && dc->lstart <= end) { + struct rb_node *node; + int err = 0; + + if (dc->len < dpolicy->granularity) + goto skip; + + if (dc->state != D_PREP) { + list_move_tail(&dc->list, &dcc->fstrim_list); + goto skip; + } + + err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); + + if (issued >= dpolicy->max_requests) { + start = dc->lstart + dc->len; + + if (err) + __remove_discard_cmd(sbi, dc); + + blk_finish_plug(&plug); + mutex_unlock(&dcc->cmd_lock); + trimmed += __wait_all_discard_cmd(sbi, NULL); + f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); + goto next; + } +skip: + node = rb_next(&dc->rb_node); + if (err) + __remove_discard_cmd(sbi, dc); + dc = rb_entry_safe(node, struct discard_cmd, rb_node); + + if (fatal_signal_pending(current)) + break; + } + + blk_finish_plug(&plug); + mutex_unlock(&dcc->cmd_lock); + + return trimmed; +} + +int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) +{ + __u64 start = F2FS_BYTES_TO_BLK(range->start); + __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; + unsigned int start_segno, end_segno; + block_t start_block, end_block; + struct cp_control cpc; + struct discard_policy dpolicy; + unsigned long long trimmed = 0; + int err = 0; + bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); + + if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) + return -EINVAL; + + if (end < MAIN_BLKADDR(sbi)) + goto out; + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { + f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); + return -EFSCORRUPTED; + } + + /* start/end segment number in main_area */ + start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); + end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : + GET_SEGNO(sbi, end); + if (need_align) { + start_segno = rounddown(start_segno, sbi->segs_per_sec); + end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; + } + + cpc.reason = CP_DISCARD; + cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); + cpc.trim_start = start_segno; + cpc.trim_end = end_segno; + + if (sbi->discard_blks == 0) + goto out; + + f2fs_down_write(&sbi->gc_lock); + err = f2fs_write_checkpoint(sbi, &cpc); + f2fs_up_write(&sbi->gc_lock); + if (err) + goto out; + + /* + * We filed discard candidates, but actually we don't need to wait for + * all of them, since they'll be issued in idle time along with runtime + * discard option. User configuration looks like using runtime discard + * or periodic fstrim instead of it. + */ + if (f2fs_realtime_discard_enable(sbi)) + goto out; + + start_block = START_BLOCK(sbi, start_segno); + end_block = START_BLOCK(sbi, end_segno + 1); + + __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); + trimmed = __issue_discard_cmd_range(sbi, &dpolicy, + start_block, end_block); + + trimmed += __wait_discard_cmd_range(sbi, &dpolicy, + start_block, end_block); +out: + if (!err) + range->len = F2FS_BLK_TO_BYTES(trimmed); + return err; +} + +int __init f2fs_create_discard_caches(void) +{ + discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", + sizeof(struct discard_entry)); + if (!discard_entry_slab) + return -ENOMEM; + + discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", + sizeof(struct discard_cmd)); + if (!discard_cmd_slab) + goto destroy_discard_entry; + return 0; + +destroy_discard_entry: + kmem_cache_destroy(discard_entry_slab); + return -ENOMEM; +} + +void f2fs_destroy_discard_caches(void) +{ + kmem_cache_destroy(discard_cmd_slab); + kmem_cache_destroy(discard_entry_slab); +} diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 4035dab1f570..1df9dd9bb400 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3638,9 +3638,38 @@ void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi); int __init f2fs_create_node_manager_caches(void); void f2fs_destroy_node_manager_caches(void); +/* + * discard.c + */ +int f2fs_issue_discard(struct f2fs_sb_info *sbi, + block_t blkstart, block_t blklen); +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi); +void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); +int create_discard_cmd_control(struct f2fs_sb_info *sbi); +void destroy_discard_cmd_control(struct f2fs_sb_info *sbi); +int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); +void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); +int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range); +bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, + struct cp_control *cpc); +void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr); +bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, + bool check_only); +void release_discard_addr(struct discard_entry *entry); +void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi); +int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen); +int __init f2fs_create_discard_caches(void); +void f2fs_destroy_discard_caches(void); + /* * segment.c */ +void __check_sit_bitmap(struct f2fs_sb_info *sbi, block_t start, block_t end); +unsigned long __find_rev_next_bit(const unsigned long *addr, + unsigned long size, unsigned long offset); +unsigned long __find_rev_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset); bool f2fs_need_SSR(struct f2fs_sb_info *sbi); int f2fs_commit_atomic_write(struct inode *inode); void f2fs_abort_atomic_write(struct inode *inode, bool clean); @@ -3652,16 +3681,11 @@ int f2fs_flush_device_cache(struct f2fs_sb_info *sbi); void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free); void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr); bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); -int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); -void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); -void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi); void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi); int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable); -void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi); int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra); bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno); void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi); @@ -3673,9 +3697,6 @@ void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, unsigned int start, unsigned int end); void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force); void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi); -int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range); -bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, - struct cp_control *cpc); struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno); void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 25ddea478fc1..b1b4d8e21365 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -27,8 +27,6 @@ #define __reverse_ffz(x) __reverse_ffs(~(x)) -static struct kmem_cache *discard_entry_slab; -static struct kmem_cache *discard_cmd_slab; static struct kmem_cache *sit_entry_set_slab; static struct kmem_cache *revoke_entry_slab; @@ -95,8 +93,8 @@ static inline unsigned long __reverse_ffs(unsigned long word) * f2fs_set_bit(0, bitmap) => 1000 0000 * f2fs_set_bit(7, bitmap) => 0000 0001 */ -static unsigned long __find_rev_next_bit(const unsigned long *addr, - unsigned long size, unsigned long offset) +unsigned long __find_rev_next_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) { const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; @@ -131,8 +129,8 @@ static unsigned long __find_rev_next_bit(const unsigned long *addr, return result - size + __reverse_ffs(tmp); } -static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, - unsigned long size, unsigned long offset) +unsigned long __find_rev_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) { const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; @@ -911,117 +909,7 @@ static unsigned int get_free_segment(struct f2fs_sb_info *sbi) return NULL_SEGNO; } -static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t lstart, - block_t start, block_t len) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct list_head *pend_list; - struct discard_cmd *dc; - - f2fs_bug_on(sbi, !len); - - pend_list = &dcc->pend_list[plist_idx(len)]; - - dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); - INIT_LIST_HEAD(&dc->list); - dc->bdev = bdev; - dc->lstart = lstart; - dc->start = start; - dc->len = len; - dc->ref = 0; - dc->state = D_PREP; - dc->queued = 0; - dc->error = 0; - init_completion(&dc->wait); - list_add_tail(&dc->list, pend_list); - spin_lock_init(&dc->lock); - dc->bio_ref = 0; - atomic_inc(&dcc->discard_cmd_cnt); - dcc->undiscard_blks += len; - - return dc; -} - -static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t lstart, - block_t start, block_t len, - struct rb_node *parent, struct rb_node **p, - bool leftmost) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_cmd *dc; - - dc = __create_discard_cmd(sbi, bdev, lstart, start, len); - - rb_link_node(&dc->rb_node, parent, p); - rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); - - return dc; -} - -static void __detach_discard_cmd(struct discard_cmd_control *dcc, - struct discard_cmd *dc) -{ - if (dc->state == D_DONE) - atomic_sub(dc->queued, &dcc->queued_discard); - - list_del(&dc->list); - rb_erase_cached(&dc->rb_node, &dcc->root); - dcc->undiscard_blks -= dc->len; - - kmem_cache_free(discard_cmd_slab, dc); - - atomic_dec(&dcc->discard_cmd_cnt); -} - -static void __remove_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_cmd *dc) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - unsigned long flags; - - trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); - - spin_lock_irqsave(&dc->lock, flags); - if (dc->bio_ref) { - spin_unlock_irqrestore(&dc->lock, flags); - return; - } - spin_unlock_irqrestore(&dc->lock, flags); - - f2fs_bug_on(sbi, dc->ref); - - if (dc->error == -EOPNOTSUPP) - dc->error = 0; - - if (dc->error) - printk_ratelimited( - "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d", - KERN_INFO, sbi->sb->s_id, - dc->lstart, dc->start, dc->len, dc->error); - __detach_discard_cmd(dcc, dc); -} - -static void f2fs_submit_discard_endio(struct bio *bio) -{ - struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; - unsigned long flags; - - spin_lock_irqsave(&dc->lock, flags); - if (!dc->error) - dc->error = blk_status_to_errno(bio->bi_status); - dc->bio_ref--; - if (!dc->bio_ref && dc->state == D_SUBMIT) { - dc->state = D_DONE; - complete_all(&dc->wait); - } - spin_unlock_irqrestore(&dc->lock, flags); - bio_put(bio); -} - -static void __check_sit_bitmap(struct f2fs_sb_info *sbi, - block_t start, block_t end) +void __check_sit_bitmap(struct f2fs_sb_info *sbi, block_t start, block_t end) { #ifdef CONFIG_F2FS_CHECK_FS struct seg_entry *sentry; @@ -1047,870 +935,6 @@ static void __check_sit_bitmap(struct f2fs_sb_info *sbi, #endif } -static void __init_discard_policy(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy, - int discard_type, unsigned int granularity) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - - /* common policy */ - dpolicy->type = discard_type; - dpolicy->sync = true; - dpolicy->ordered = false; - dpolicy->granularity = granularity; - - dpolicy->max_requests = dcc->max_discard_request; - dpolicy->io_aware_gran = MAX_PLIST_NUM; - dpolicy->timeout = false; - - if (discard_type == DPOLICY_BG) { - dpolicy->min_interval = dcc->min_discard_issue_time; - dpolicy->mid_interval = dcc->mid_discard_issue_time; - dpolicy->max_interval = dcc->max_discard_issue_time; - dpolicy->io_aware = true; - dpolicy->sync = false; - dpolicy->ordered = true; - if (utilization(sbi) > dcc->discard_urgent_util) { - dpolicy->granularity = MIN_DISCARD_GRANULARITY; - if (atomic_read(&dcc->discard_cmd_cnt)) - dpolicy->max_interval = - dcc->min_discard_issue_time; - } - } else if (discard_type == DPOLICY_FORCE) { - dpolicy->min_interval = dcc->min_discard_issue_time; - dpolicy->mid_interval = dcc->mid_discard_issue_time; - dpolicy->max_interval = dcc->max_discard_issue_time; - dpolicy->io_aware = false; - } else if (discard_type == DPOLICY_FSTRIM) { - dpolicy->io_aware = false; - } else if (discard_type == DPOLICY_UMOUNT) { - dpolicy->io_aware = false; - /* we need to issue all to keep CP_TRIMMED_FLAG */ - dpolicy->granularity = MIN_DISCARD_GRANULARITY; - dpolicy->timeout = true; - } -} - -static void __update_discard_tree_range(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t lstart, - block_t start, block_t len); -/* this function is copied from blkdev_issue_discard from block/blk-lib.c */ -static int __submit_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy, - struct discard_cmd *dc, - unsigned int *issued) -{ - struct block_device *bdev = dc->bdev; - unsigned int max_discard_blocks = - SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? - &(dcc->fstrim_list) : &(dcc->wait_list); - blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; - block_t lstart, start, len, total_len; - int err = 0; - - if (dc->state != D_PREP) - return 0; - - if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) - return 0; - - trace_f2fs_issue_discard(bdev, dc->start, dc->len); - - lstart = dc->lstart; - start = dc->start; - len = dc->len; - total_len = len; - - dc->len = 0; - - while (total_len && *issued < dpolicy->max_requests && !err) { - struct bio *bio = NULL; - unsigned long flags; - bool last = true; - - if (len > max_discard_blocks) { - len = max_discard_blocks; - last = false; - } - - (*issued)++; - if (*issued == dpolicy->max_requests) - last = true; - - dc->len += len; - - if (time_to_inject(sbi, FAULT_DISCARD)) { - f2fs_show_injection_info(sbi, FAULT_DISCARD); - err = -EIO; - } else { - err = __blkdev_issue_discard(bdev, - SECTOR_FROM_BLOCK(start), - SECTOR_FROM_BLOCK(len), - GFP_NOFS, &bio); - } - if (err) { - spin_lock_irqsave(&dc->lock, flags); - if (dc->state == D_PARTIAL) - dc->state = D_SUBMIT; - spin_unlock_irqrestore(&dc->lock, flags); - - break; - } - - f2fs_bug_on(sbi, !bio); - - /* - * should keep before submission to avoid D_DONE - * right away - */ - spin_lock_irqsave(&dc->lock, flags); - if (last) - dc->state = D_SUBMIT; - else - dc->state = D_PARTIAL; - dc->bio_ref++; - spin_unlock_irqrestore(&dc->lock, flags); - - atomic_inc(&dcc->queued_discard); - dc->queued++; - list_move_tail(&dc->list, wait_list); - - /* sanity check on discard range */ - __check_sit_bitmap(sbi, lstart, lstart + len); - - bio->bi_private = dc; - bio->bi_end_io = f2fs_submit_discard_endio; - bio->bi_opf |= flag; - submit_bio(bio); - - atomic_inc(&dcc->issued_discard); - - f2fs_update_iostat(sbi, NULL, FS_DISCARD, len * F2FS_BLKSIZE); - - lstart += len; - start += len; - total_len -= len; - len = total_len; - } - - if (!err && len) { - dcc->undiscard_blks -= len; - __update_discard_tree_range(sbi, bdev, lstart, start, len); - } - return err; -} - -static void __insert_discard_tree(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t lstart, - block_t start, block_t len, - struct rb_node **insert_p, - struct rb_node *insert_parent) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct rb_node **p; - struct rb_node *parent = NULL; - bool leftmost = true; - - if (insert_p && insert_parent) { - parent = insert_parent; - p = insert_p; - goto do_insert; - } - - p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, - lstart, &leftmost); -do_insert: - __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, - p, leftmost); -} - -static void __relocate_discard_cmd(struct discard_cmd_control *dcc, - struct discard_cmd *dc) -{ - list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); -} - -static void __punch_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_cmd *dc, block_t blkaddr) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_info di = dc->di; - bool modified = false; - - if (dc->state == D_DONE || dc->len == 1) { - __remove_discard_cmd(sbi, dc); - return; - } - - dcc->undiscard_blks -= di.len; - - if (blkaddr > di.lstart) { - dc->len = blkaddr - dc->lstart; - dcc->undiscard_blks += dc->len; - __relocate_discard_cmd(dcc, dc); - modified = true; - } - - if (blkaddr < di.lstart + di.len - 1) { - if (modified) { - __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, - di.start + blkaddr + 1 - di.lstart, - di.lstart + di.len - 1 - blkaddr, - NULL, NULL); - } else { - dc->lstart++; - dc->len--; - dc->start++; - dcc->undiscard_blks += dc->len; - __relocate_discard_cmd(dcc, dc); - } - } -} - -static void __update_discard_tree_range(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t lstart, - block_t start, block_t len) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_cmd *prev_dc = NULL, *next_dc = NULL; - struct discard_cmd *dc; - struct discard_info di = {0}; - struct rb_node **insert_p = NULL, *insert_parent = NULL; - unsigned int max_discard_blocks = - SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); - block_t end = lstart + len; - - dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, - NULL, lstart, - (struct rb_entry **)&prev_dc, - (struct rb_entry **)&next_dc, - &insert_p, &insert_parent, true, NULL); - if (dc) - prev_dc = dc; - - if (!prev_dc) { - di.lstart = lstart; - di.len = next_dc ? next_dc->lstart - lstart : len; - di.len = min(di.len, len); - di.start = start; - } - - while (1) { - struct rb_node *node; - bool merged = false; - struct discard_cmd *tdc = NULL; - - if (prev_dc) { - di.lstart = prev_dc->lstart + prev_dc->len; - if (di.lstart < lstart) - di.lstart = lstart; - if (di.lstart >= end) - break; - - if (!next_dc || next_dc->lstart > end) - di.len = end - di.lstart; - else - di.len = next_dc->lstart - di.lstart; - di.start = start + di.lstart - lstart; - } - - if (!di.len) - goto next; - - if (prev_dc && prev_dc->state == D_PREP && - prev_dc->bdev == bdev && - __is_discard_back_mergeable(&di, &prev_dc->di, - max_discard_blocks)) { - prev_dc->di.len += di.len; - dcc->undiscard_blks += di.len; - __relocate_discard_cmd(dcc, prev_dc); - di = prev_dc->di; - tdc = prev_dc; - merged = true; - } - - if (next_dc && next_dc->state == D_PREP && - next_dc->bdev == bdev && - __is_discard_front_mergeable(&di, &next_dc->di, - max_discard_blocks)) { - next_dc->di.lstart = di.lstart; - next_dc->di.len += di.len; - next_dc->di.start = di.start; - dcc->undiscard_blks += di.len; - __relocate_discard_cmd(dcc, next_dc); - if (tdc) - __remove_discard_cmd(sbi, tdc); - merged = true; - } - - if (!merged) { - __insert_discard_tree(sbi, bdev, di.lstart, di.start, - di.len, NULL, NULL); - } - next: - prev_dc = next_dc; - if (!prev_dc) - break; - - node = rb_next(&prev_dc->rb_node); - next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); - } -} - -static void __queue_discard_cmd(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t blkstart, block_t blklen) -{ - block_t lblkstart = blkstart; - - if (!f2fs_bdev_support_discard(bdev)) - return; - - trace_f2fs_queue_discard(bdev, blkstart, blklen); - - if (f2fs_is_multi_device(sbi)) { - int devi = f2fs_target_device_index(sbi, blkstart); - - blkstart -= FDEV(devi).start_blk; - } - mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); - __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); - mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); -} - -static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_cmd *prev_dc = NULL, *next_dc = NULL; - struct rb_node **insert_p = NULL, *insert_parent = NULL; - struct discard_cmd *dc; - struct blk_plug plug; - unsigned int pos = dcc->next_pos; - unsigned int issued = 0; - bool io_interrupted = false; - - mutex_lock(&dcc->cmd_lock); - dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, - NULL, pos, - (struct rb_entry **)&prev_dc, - (struct rb_entry **)&next_dc, - &insert_p, &insert_parent, true, NULL); - if (!dc) - dc = next_dc; - - blk_start_plug(&plug); - - while (dc) { - struct rb_node *node; - int err = 0; - - if (dc->state != D_PREP) - goto next; - - if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { - io_interrupted = true; - break; - } - - dcc->next_pos = dc->lstart + dc->len; - err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); - - if (issued >= dpolicy->max_requests) - break; -next: - node = rb_next(&dc->rb_node); - if (err) - __remove_discard_cmd(sbi, dc); - dc = rb_entry_safe(node, struct discard_cmd, rb_node); - } - - blk_finish_plug(&plug); - - if (!dc) - dcc->next_pos = 0; - - mutex_unlock(&dcc->cmd_lock); - - if (!issued && io_interrupted) - issued = -1; - - return issued; -} -static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy); - -static int __issue_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct list_head *pend_list; - struct discard_cmd *dc, *tmp; - struct blk_plug plug; - int i, issued; - bool io_interrupted = false; - - if (dpolicy->timeout) - f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); - -retry: - issued = 0; - for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { - if (dpolicy->timeout && - f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) - break; - - if (i + 1 < dpolicy->granularity) - break; - - if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) - return __issue_discard_cmd_orderly(sbi, dpolicy); - - pend_list = &dcc->pend_list[i]; - - mutex_lock(&dcc->cmd_lock); - if (list_empty(pend_list)) - goto next; - if (unlikely(dcc->rbtree_check)) - f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, - &dcc->root, false)); - blk_start_plug(&plug); - list_for_each_entry_safe(dc, tmp, pend_list, list) { - f2fs_bug_on(sbi, dc->state != D_PREP); - - if (dpolicy->timeout && - f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) - break; - - if (dpolicy->io_aware && i < dpolicy->io_aware_gran && - !is_idle(sbi, DISCARD_TIME)) { - io_interrupted = true; - break; - } - - __submit_discard_cmd(sbi, dpolicy, dc, &issued); - - if (issued >= dpolicy->max_requests) - break; - } - blk_finish_plug(&plug); -next: - mutex_unlock(&dcc->cmd_lock); - - if (issued >= dpolicy->max_requests || io_interrupted) - break; - } - - if (dpolicy->type == DPOLICY_UMOUNT && issued) { - __wait_all_discard_cmd(sbi, dpolicy); - goto retry; - } - - if (!issued && io_interrupted) - issued = -1; - - return issued; -} - -static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct list_head *pend_list; - struct discard_cmd *dc, *tmp; - int i; - bool dropped = false; - - mutex_lock(&dcc->cmd_lock); - for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { - pend_list = &dcc->pend_list[i]; - list_for_each_entry_safe(dc, tmp, pend_list, list) { - f2fs_bug_on(sbi, dc->state != D_PREP); - __remove_discard_cmd(sbi, dc); - dropped = true; - } - } - mutex_unlock(&dcc->cmd_lock); - - return dropped; -} - -void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) -{ - __drop_discard_cmd(sbi); -} - -static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, - struct discard_cmd *dc) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - unsigned int len = 0; - - wait_for_completion_io(&dc->wait); - mutex_lock(&dcc->cmd_lock); - f2fs_bug_on(sbi, dc->state != D_DONE); - dc->ref--; - if (!dc->ref) { - if (!dc->error) - len = dc->len; - __remove_discard_cmd(sbi, dc); - } - mutex_unlock(&dcc->cmd_lock); - - return len; -} - -static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy, - block_t start, block_t end) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? - &(dcc->fstrim_list) : &(dcc->wait_list); - struct discard_cmd *dc = NULL, *iter, *tmp; - unsigned int trimmed = 0; - -next: - dc = NULL; - - mutex_lock(&dcc->cmd_lock); - list_for_each_entry_safe(iter, tmp, wait_list, list) { - if (iter->lstart + iter->len <= start || end <= iter->lstart) - continue; - if (iter->len < dpolicy->granularity) - continue; - if (iter->state == D_DONE && !iter->ref) { - wait_for_completion_io(&iter->wait); - if (!iter->error) - trimmed += iter->len; - __remove_discard_cmd(sbi, iter); - } else { - iter->ref++; - dc = iter; - break; - } - } - mutex_unlock(&dcc->cmd_lock); - - if (dc) { - trimmed += __wait_one_discard_bio(sbi, dc); - goto next; - } - - return trimmed; -} - -static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy) -{ - struct discard_policy dp; - unsigned int discard_blks; - - if (dpolicy) - return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); - - /* wait all */ - __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); - discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); - __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); - discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); - - return discard_blks; -} - -/* This should be covered by global mutex, &sit_i->sentry_lock */ -static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_cmd *dc; - bool need_wait = false; - - mutex_lock(&dcc->cmd_lock); - dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root, - NULL, blkaddr); - if (dc) { - if (dc->state == D_PREP) { - __punch_discard_cmd(sbi, dc, blkaddr); - } else { - dc->ref++; - need_wait = true; - } - } - mutex_unlock(&dcc->cmd_lock); - - if (need_wait) - __wait_one_discard_bio(sbi, dc); -} - -void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - - if (dcc && dcc->f2fs_issue_discard) { - struct task_struct *discard_thread = dcc->f2fs_issue_discard; - - dcc->f2fs_issue_discard = NULL; - kthread_stop(discard_thread); - } -} - -/* This comes from f2fs_put_super */ -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_policy dpolicy; - bool dropped; - - if (!atomic_read(&dcc->discard_cmd_cnt)) - return false; - - __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, - dcc->discard_granularity); - __issue_discard_cmd(sbi, &dpolicy); - dropped = __drop_discard_cmd(sbi); - - /* just to make sure there is no pending discard commands */ - __wait_all_discard_cmd(sbi, NULL); - - f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); - return dropped; -} - -static int issue_discard_thread(void *data) -{ - struct f2fs_sb_info *sbi = data; - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - wait_queue_head_t *q = &dcc->discard_wait_queue; - struct discard_policy dpolicy; - unsigned int wait_ms = dcc->min_discard_issue_time; - int issued; - - set_freezable(); - - do { - wait_event_interruptible_timeout(*q, - kthread_should_stop() || freezing(current) || - dcc->discard_wake, - msecs_to_jiffies(wait_ms)); - - if (sbi->gc_mode == GC_URGENT_HIGH || - !f2fs_available_free_memory(sbi, DISCARD_CACHE)) - __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); - else - __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, - dcc->discard_granularity); - - if (dcc->discard_wake) - dcc->discard_wake = 0; - - /* clean up pending candidates before going to sleep */ - if (atomic_read(&dcc->queued_discard)) - __wait_all_discard_cmd(sbi, NULL); - - if (try_to_freeze()) - continue; - if (f2fs_readonly(sbi->sb)) - continue; - if (kthread_should_stop()) - return 0; - if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || - !atomic_read(&dcc->discard_cmd_cnt)) { - wait_ms = dpolicy.max_interval; - continue; - } - - sb_start_intwrite(sbi->sb); - - issued = __issue_discard_cmd(sbi, &dpolicy); - if (issued > 0) { - __wait_all_discard_cmd(sbi, &dpolicy); - wait_ms = dpolicy.min_interval; - } else if (issued == -1) { - wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); - if (!wait_ms) - wait_ms = dpolicy.mid_interval; - } else { - wait_ms = dpolicy.max_interval; - } - if (!atomic_read(&dcc->discard_cmd_cnt)) - wait_ms = dpolicy.max_interval; - - sb_end_intwrite(sbi->sb); - - } while (!kthread_should_stop()); - return 0; -} - -#ifdef CONFIG_BLK_DEV_ZONED -static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t blkstart, block_t blklen) -{ - sector_t sector, nr_sects; - block_t lblkstart = blkstart; - int devi = 0; - - if (f2fs_is_multi_device(sbi)) { - devi = f2fs_target_device_index(sbi, blkstart); - if (blkstart < FDEV(devi).start_blk || - blkstart > FDEV(devi).end_blk) { - f2fs_err(sbi, "Invalid block %x", blkstart); - return -EIO; - } - blkstart -= FDEV(devi).start_blk; - } - - /* For sequential zones, reset the zone write pointer */ - if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { - sector = SECTOR_FROM_BLOCK(blkstart); - nr_sects = SECTOR_FROM_BLOCK(blklen); - - if (sector & (bdev_zone_sectors(bdev) - 1) || - nr_sects != bdev_zone_sectors(bdev)) { - f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", - devi, sbi->s_ndevs ? FDEV(devi).path : "", - blkstart, blklen); - return -EIO; - } - trace_f2fs_issue_reset_zone(bdev, blkstart); - return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, - sector, nr_sects, GFP_NOFS); - } - - /* For conventional zones, use regular discard if supported */ - __queue_discard_cmd(sbi, bdev, lblkstart, blklen); - return 0; -} -#endif - -static int __issue_discard_async(struct f2fs_sb_info *sbi, - struct block_device *bdev, block_t blkstart, block_t blklen) -{ -#ifdef CONFIG_BLK_DEV_ZONED - if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) - return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); -#endif - __queue_discard_cmd(sbi, bdev, blkstart, blklen); - return 0; -} - -static int f2fs_issue_discard(struct f2fs_sb_info *sbi, - block_t blkstart, block_t blklen) -{ - sector_t start = blkstart, len = 0; - struct block_device *bdev; - struct seg_entry *se; - unsigned int offset; - block_t i; - int err = 0; - - bdev = f2fs_target_device(sbi, blkstart, NULL); - - for (i = blkstart; i < blkstart + blklen; i++, len++) { - if (i != start) { - struct block_device *bdev2 = - f2fs_target_device(sbi, i, NULL); - - if (bdev2 != bdev) { - err = __issue_discard_async(sbi, bdev, - start, len); - if (err) - return err; - bdev = bdev2; - start = i; - len = 0; - } - } - - se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); - offset = GET_BLKOFF_FROM_SEG0(sbi, i); - - if (f2fs_block_unit_discard(sbi) && - !f2fs_test_and_set_bit(offset, se->discard_map)) - sbi->discard_blks--; - } - - if (len) - err = __issue_discard_async(sbi, bdev, start, len); - return err; -} - -static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, - bool check_only) -{ - int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); - int max_blocks = sbi->blocks_per_seg; - struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); - unsigned long *cur_map = (unsigned long *)se->cur_valid_map; - unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; - unsigned long *discard_map = (unsigned long *)se->discard_map; - unsigned long *dmap = SIT_I(sbi)->tmp_map; - unsigned int start = 0, end = -1; - bool force = (cpc->reason & CP_DISCARD); - struct discard_entry *de = NULL; - struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; - int i; - - if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) || - !f2fs_block_unit_discard(sbi)) - return false; - - if (!force) { - if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || - SM_I(sbi)->dcc_info->nr_discards >= - SM_I(sbi)->dcc_info->max_discards) - return false; - } - - /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ - for (i = 0; i < entries; i++) - dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : - (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; - - while (force || SM_I(sbi)->dcc_info->nr_discards <= - SM_I(sbi)->dcc_info->max_discards) { - start = __find_rev_next_bit(dmap, max_blocks, end + 1); - if (start >= max_blocks) - break; - - end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); - if (force && start && end != max_blocks - && (end - start) < cpc->trim_minlen) - continue; - - if (check_only) - return true; - - if (!de) { - de = f2fs_kmem_cache_alloc(discard_entry_slab, - GFP_F2FS_ZERO, true, NULL); - de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); - list_add_tail(&de->list, head); - } - - for (i = start; i < end; i++) - __set_bit_le(i, (void *)de->discard_map); - - SM_I(sbi)->dcc_info->nr_discards += end - start; - } - return false; -} - -static void release_discard_addr(struct discard_entry *entry) -{ - list_del(&entry->list); - kmem_cache_free(discard_entry_slab, entry); -} - -void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) -{ - struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); - struct discard_entry *entry, *this; - - /* drop caches */ - list_for_each_entry_safe(entry, this, head, list) - release_discard_addr(entry); -} - /* * Should call f2fs_clear_prefree_segments after checkpoint is done. */ @@ -2033,98 +1057,6 @@ void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, wake_up_discard_thread(sbi, false); } -int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) -{ - dev_t dev = sbi->sb->s_bdev->bd_dev; - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - int err = 0; - - if (!f2fs_realtime_discard_enable(sbi)) - return 0; - - dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, - "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); - if (IS_ERR(dcc->f2fs_issue_discard)) { - err = PTR_ERR(dcc->f2fs_issue_discard); - dcc->f2fs_issue_discard = NULL; - } - - return err; -} - -static int create_discard_cmd_control(struct f2fs_sb_info *sbi) -{ - struct discard_cmd_control *dcc; - int err = 0, i; - - if (SM_I(sbi)->dcc_info) { - dcc = SM_I(sbi)->dcc_info; - goto init_thread; - } - - dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); - if (!dcc) - return -ENOMEM; - - dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; - dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; - if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) - dcc->discard_granularity = sbi->blocks_per_seg; - else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) - dcc->discard_granularity = BLKS_PER_SEC(sbi); - - INIT_LIST_HEAD(&dcc->entry_list); - for (i = 0; i < MAX_PLIST_NUM; i++) - INIT_LIST_HEAD(&dcc->pend_list[i]); - INIT_LIST_HEAD(&dcc->wait_list); - INIT_LIST_HEAD(&dcc->fstrim_list); - mutex_init(&dcc->cmd_lock); - atomic_set(&dcc->issued_discard, 0); - atomic_set(&dcc->queued_discard, 0); - atomic_set(&dcc->discard_cmd_cnt, 0); - dcc->nr_discards = 0; - dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; - dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; - dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; - dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; - dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; - dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; - dcc->undiscard_blks = 0; - dcc->next_pos = 0; - dcc->root = RB_ROOT_CACHED; - dcc->rbtree_check = false; - - init_waitqueue_head(&dcc->discard_wait_queue); - SM_I(sbi)->dcc_info = dcc; -init_thread: - err = f2fs_start_discard_thread(sbi); - if (err) { - kfree(dcc); - SM_I(sbi)->dcc_info = NULL; - } - - return err; -} - -static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - - if (!dcc) - return; - - f2fs_stop_discard_thread(sbi); - - /* - * Recovery can cache discard commands, so in error path of - * fill_super(), it needs to give a chance to handle them. - */ - f2fs_issue_discard_timeout(sbi); - - kfree(dcc); - SM_I(sbi)->dcc_info = NULL; -} - static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) { struct sit_info *sit_i = SIT_I(sbi); @@ -2953,167 +1885,6 @@ void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) f2fs_up_read(&SM_I(sbi)->curseg_lock); } -bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, - struct cp_control *cpc) -{ - __u64 trim_start = cpc->trim_start; - bool has_candidate = false; - - down_write(&SIT_I(sbi)->sentry_lock); - for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { - if (add_discard_addrs(sbi, cpc, true)) { - has_candidate = true; - break; - } - } - up_write(&SIT_I(sbi)->sentry_lock); - - cpc->trim_start = trim_start; - return has_candidate; -} - -static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, - struct discard_policy *dpolicy, - unsigned int start, unsigned int end) -{ - struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; - struct discard_cmd *prev_dc = NULL, *next_dc = NULL; - struct rb_node **insert_p = NULL, *insert_parent = NULL; - struct discard_cmd *dc; - struct blk_plug plug; - int issued; - unsigned int trimmed = 0; - -next: - issued = 0; - - mutex_lock(&dcc->cmd_lock); - if (unlikely(dcc->rbtree_check)) - f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, - &dcc->root, false)); - - dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, - NULL, start, - (struct rb_entry **)&prev_dc, - (struct rb_entry **)&next_dc, - &insert_p, &insert_parent, true, NULL); - if (!dc) - dc = next_dc; - - blk_start_plug(&plug); - - while (dc && dc->lstart <= end) { - struct rb_node *node; - int err = 0; - - if (dc->len < dpolicy->granularity) - goto skip; - - if (dc->state != D_PREP) { - list_move_tail(&dc->list, &dcc->fstrim_list); - goto skip; - } - - err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); - - if (issued >= dpolicy->max_requests) { - start = dc->lstart + dc->len; - - if (err) - __remove_discard_cmd(sbi, dc); - - blk_finish_plug(&plug); - mutex_unlock(&dcc->cmd_lock); - trimmed += __wait_all_discard_cmd(sbi, NULL); - f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); - goto next; - } -skip: - node = rb_next(&dc->rb_node); - if (err) - __remove_discard_cmd(sbi, dc); - dc = rb_entry_safe(node, struct discard_cmd, rb_node); - - if (fatal_signal_pending(current)) - break; - } - - blk_finish_plug(&plug); - mutex_unlock(&dcc->cmd_lock); - - return trimmed; -} - -int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) -{ - __u64 start = F2FS_BYTES_TO_BLK(range->start); - __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; - unsigned int start_segno, end_segno; - block_t start_block, end_block; - struct cp_control cpc; - struct discard_policy dpolicy; - unsigned long long trimmed = 0; - int err = 0; - bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); - - if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) - return -EINVAL; - - if (end < MAIN_BLKADDR(sbi)) - goto out; - - if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { - f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); - return -EFSCORRUPTED; - } - - /* start/end segment number in main_area */ - start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); - end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : - GET_SEGNO(sbi, end); - if (need_align) { - start_segno = rounddown(start_segno, sbi->segs_per_sec); - end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; - } - - cpc.reason = CP_DISCARD; - cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); - cpc.trim_start = start_segno; - cpc.trim_end = end_segno; - - if (sbi->discard_blks == 0) - goto out; - - f2fs_down_write(&sbi->gc_lock); - err = f2fs_write_checkpoint(sbi, &cpc); - f2fs_up_write(&sbi->gc_lock); - if (err) - goto out; - - /* - * We filed discard candidates, but actually we don't need to wait for - * all of them, since they'll be issued in idle time along with runtime - * discard option. User configuration looks like using runtime discard - * or periodic fstrim instead of it. - */ - if (f2fs_realtime_discard_enable(sbi)) - goto out; - - start_block = START_BLOCK(sbi, start_segno); - end_block = START_BLOCK(sbi, end_segno + 1); - - __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); - trimmed = __issue_discard_cmd_range(sbi, &dpolicy, - start_block, end_block); - - trimmed += __wait_discard_cmd_range(sbi, &dpolicy, - start_block, end_block); -out: - if (!err) - range->len = F2FS_BLK_TO_BYTES(trimmed); - return err; -} - static bool __has_curseg_space(struct f2fs_sb_info *sbi, struct curseg_info *curseg) { @@ -5287,41 +4058,34 @@ void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) int __init f2fs_create_segment_manager_caches(void) { - discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", - sizeof(struct discard_entry)); - if (!discard_entry_slab) - goto fail; - - discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", - sizeof(struct discard_cmd)); - if (!discard_cmd_slab) - goto destroy_discard_entry; + int ret; sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", sizeof(struct sit_entry_set)); if (!sit_entry_set_slab) - goto destroy_discard_cmd; + return -ENOMEM; revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry", sizeof(struct revoke_entry)); if (!revoke_entry_slab) goto destroy_sit_entry_set; - return 0; + ret = f2fs_create_discard_caches(); + if (ret) + goto destroy_revoke_entry; + + return ret; + +destroy_revoke_entry: + kmem_cache_destroy(revoke_entry_slab); destroy_sit_entry_set: kmem_cache_destroy(sit_entry_set_slab); -destroy_discard_cmd: - kmem_cache_destroy(discard_cmd_slab); -destroy_discard_entry: - kmem_cache_destroy(discard_entry_slab); -fail: return -ENOMEM; } void f2fs_destroy_segment_manager_caches(void) { + f2fs_destroy_discard_caches(); kmem_cache_destroy(sit_entry_set_slab); - kmem_cache_destroy(discard_cmd_slab); - kmem_cache_destroy(discard_entry_slab); kmem_cache_destroy(revoke_entry_slab); } -- 2.25.1 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
