From: Nanzhe <[email protected]> Large folio can contain multiple dirty ranges. Add a folio-based writeback path for large-folio mapping files and keep the legacy f2fs_write_cache_pages() path unchanged for non large-folio mapping files.
Signed-off-by: Nanzhe <[email protected]> --- fs/f2fs/data.c | 400 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 394 insertions(+), 6 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 390b50eff260..2de8c7963080 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -352,7 +352,9 @@ static void f2fs_write_end_io(struct bio *bio) bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; + unsigned int nr_pages = fi.length >> PAGE_SHIFT; enum count_type type; + bool finished = true; if (fscrypt_is_bounce_folio(folio)) { struct folio *io_folio = folio; @@ -362,7 +364,7 @@ static void f2fs_write_end_io(struct bio *bio) } #ifdef CONFIG_F2FS_FS_COMPRESSION - if (f2fs_is_compressed_page(folio)) { + if (!folio_test_large(folio) && f2fs_is_compressed_page(folio)) { f2fs_compress_write_end_io(bio, folio); continue; } @@ -383,11 +385,19 @@ static void f2fs_write_end_io(struct bio *bio) folio->index, NODE_TYPE_REGULAR, true); f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); } + if (folio_has_ffs(folio)) { + struct f2fs_folio_state *ffs = folio->private; + + finished = atomic_sub_and_test(nr_pages, + &ffs->write_pages_pending); + } + + while (nr_pages--) + dec_page_count(sbi, type); + if (f2fs_in_warm_node_list(folio)) f2fs_del_fsync_node_entry(sbi, folio); - dec_page_count(sbi, type); - /* * we should access sbi before folio_end_writeback() to * avoid racing w/ kill_f2fs_super() @@ -396,8 +406,10 @@ static void f2fs_write_end_io(struct bio *bio) wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); - folio_clear_f2fs_gcing(folio); - folio_end_writeback(folio); + if (finished) { + folio_clear_f2fs_gcing(folio); + folio_end_writeback(folio); + } } bio_put(bio); @@ -2605,6 +2617,87 @@ static void ffs_mark_subrange_dirty(struct folio *folio, spin_unlock_irqrestore(&ffs->state_lock, flags); } +static bool __ffs_clear_subrange_dirty(struct folio *folio, + struct f2fs_folio_state *ffs, size_t offset, size_t len) +{ + unsigned int nr_subpages = folio_nr_pages(folio); + unsigned int start, end; + + start = offset >> PAGE_SHIFT; + end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + end = min(end, nr_subpages); + + bitmap_clear(ffs->state, nr_subpages + start, end - start); + return find_next_bit(ffs->state, 2 * nr_subpages, nr_subpages) < + 2 * nr_subpages; +} + +static void ffs_clear_subrange_dirty(struct folio *folio, + size_t offset, size_t len) +{ + struct f2fs_folio_state *ffs; + unsigned long flags; + + if (!folio_has_ffs(folio)) + return; + + ffs = folio->private; + spin_lock_irqsave(&ffs->state_lock, flags); + __ffs_clear_subrange_dirty(folio, ffs, offset, len); + spin_unlock_irqrestore(&ffs->state_lock, flags); +} + +static unsigned int ffs_next_dirty_subpage(struct f2fs_folio_state *ffs, + const struct folio *folio, unsigned int start, + unsigned int end) +{ + unsigned int nr_subpages = folio_nr_pages(folio); + + return find_next_bit(ffs->state, nr_subpages + end + 1, + nr_subpages + start) - nr_subpages; +} + +static unsigned int ffs_next_clean_subpage(struct f2fs_folio_state *ffs, + const struct folio *folio, unsigned int start, + unsigned int end) +{ + unsigned int nr_subpages = folio_nr_pages(folio); + + return find_next_zero_bit(ffs->state, nr_subpages + end + 1, + nr_subpages + start) - nr_subpages; +} + +static unsigned int ffs_find_dirty_range(struct folio *folio, + u64 *range_start, u64 range_end) +{ + struct f2fs_folio_state *ffs; + unsigned int start, end, nr_pages; + + if (*range_start >= range_end) + return 0; + + if (!folio_has_ffs(folio)) + return range_end - *range_start; + + ffs = folio->private; + start = offset_in_folio(folio, *range_start) >> PAGE_SHIFT; + end = DIV_ROUND_UP(min_not_zero(offset_in_folio(folio, range_end), + folio_size(folio)), PAGE_SIZE) - 1; + + start = ffs_next_dirty_subpage(ffs, folio, start, end); + if (start > end) + return 0; + + if (start == end) + nr_pages = 1; + else + nr_pages = ffs_next_clean_subpage(ffs, folio, + start + 1, end) - start; + + *range_start = folio_pos(folio) + ((u64)start << PAGE_SHIFT); + return (u64)nr_pages << PAGE_SHIFT; +} + static bool f2fs_find_next_need_read_block(const struct folio *folio, size_t orig_off, size_t *need_off, size_t len) @@ -3452,6 +3545,137 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) return err; } +static int f2fs_write_single_data_folio(struct folio *folio, int *submitted, + struct writeback_control *wbc, + enum iostat_type io_type, + u64 start, u64 end) +{ + struct inode *inode = folio->mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool atomic_commit = f2fs_is_atomic_file(inode) && + folio_test_f2fs_atomic(folio); + struct inode *dn_inode = atomic_commit ? + F2FS_I(inode)->cow_inode : inode; + u64 pos = folio_pos(folio); + pgoff_t start_idx = (start - pos) >> PAGE_SHIFT; + pgoff_t end_idx = (end - 1 - pos) >> PAGE_SHIFT; + int local_submitted = 0; + int err = 0; + + for (pgoff_t i = start_idx; i <= end_idx; i++) { + struct dnode_of_data dn; + struct node_info ni; + pgoff_t data_idx = folio->index + i; + bool ipu_force = false; + struct f2fs_io_info fio = { + .sbi = sbi, + .ino = inode->i_ino, + .type = DATA, + .op = REQ_OP_WRITE, + .op_flags = wbc_to_write_flags(wbc), + .old_blkaddr = NULL_ADDR, + .folio = folio, + .idx = i, + .cnt = 1, + .encrypted_page = NULL, + .submitted = 0, + .need_lock = LOCK_DONE, + .meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0, + .io_type = io_type, + .io_wbc = wbc, + }; + + if (folio_has_ffs(folio)) { + struct f2fs_folio_state *ffs = folio->private; + + atomic_inc(&ffs->write_pages_pending); + } + + set_new_dnode(&dn, dn_inode, NULL, NULL, 0); + + if (!atomic_commit && need_inplace_update(&fio) && + f2fs_lookup_read_extent_cache_block(inode, data_idx, + &fio.old_blkaddr)) { + if (!f2fs_is_valid_blkaddr(sbi, fio.old_blkaddr, + DATA_GENERIC_ENHANCE)) { + err = -EFSCORRUPTED; + goto rollback; + } + ipu_force = true; + goto got_it; + } + + err = f2fs_get_dnode_of_data(&dn, data_idx, LOOKUP_NODE); + if (err) + goto rollback; + + fio.old_blkaddr = dn.data_blkaddr; + +got_it: + if (__is_valid_data_blkaddr(fio.old_blkaddr) && + !f2fs_is_valid_blkaddr(sbi, fio.old_blkaddr, + DATA_GENERIC_ENHANCE)) { + err = -EFSCORRUPTED; + goto rollback; + } + + if (fio.meta_gc) + f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); + + if (!atomic_commit && (ipu_force || + (__is_valid_data_blkaddr(fio.old_blkaddr) && + need_inplace_update(&fio)))) { + err = f2fs_encrypt_one_page(&fio); + if (err) + goto rollback; + + f2fs_put_dnode(&dn); + err = f2fs_inplace_write_data(&fio); + if (err) { + if (fscrypt_inode_uses_fs_layer_crypto(inode)) + fscrypt_finalize_bounce_page( + &fio.encrypted_page); + goto rollback_no_dnode; + } + + local_submitted++; + set_inode_flag(inode, FI_UPDATE_WRITE); + continue; + } + + err = f2fs_get_node_info(sbi, dn.nid, &ni, false); + if (err) + goto rollback; + + fio.version = ni.version; + + err = f2fs_encrypt_one_page(&fio); + if (err) + goto rollback; + + f2fs_outplace_write_data(&dn, &fio); + local_submitted++; + set_inode_flag(inode, FI_APPEND_WRITE); + trace_f2fs_do_write_data_page(folio, OPU); + f2fs_put_dnode(&dn); + continue; + +rollback: + f2fs_put_dnode(&dn); +rollback_no_dnode: + if (folio_has_ffs(folio)) { + struct f2fs_folio_state *ffs = folio->private; + + atomic_dec(&ffs->write_pages_pending); + } + break; + } + + if (submitted) + *submitted = local_submitted; + return err; +} + int f2fs_write_single_data_page(struct folio *folio, int *submitted, struct bio **bio, sector_t *last_block, @@ -3900,6 +4124,167 @@ static int f2fs_write_cache_pages(struct address_space *mapping, return ret; } +static int f2fs_write_cache_folios(struct address_space *mapping, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + struct folio *folio = NULL; + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + u64 pos = 0; + u64 end_pos = 0; + u32 r_len = 0; + int err = 0; + int submitted = 0; + int nwritten = 0; + bool op_locked = false; + bool next = false; + bool retry = false; + + if (get_dirty_pages(inode) <= SM_I(sbi)->min_hot_blocks) + set_inode_flag(inode, FI_HOT_DATA); + else + clear_inode_flag(inode, FI_HOT_DATA); + + while ((folio = writeback_iter(mapping, wbc, folio, &err))) { + struct f2fs_folio_state *ffs = NULL; + u64 isize; + size_t poff; + pgoff_t end_index; + bool verity_in_progress; + int folio_submitted = 0; + bool bias_added = false; + + submitted = 0; + next = true; + retry = false; + + if (atomic_read(&sbi->wb_sync_req[DATA]) && + wbc->sync_mode == WB_SYNC_NONE) { + folio_redirty_for_writepage(wbc, folio); + next = false; + goto retry_out; + } +retry: + pos = folio_pos(folio); + end_pos = pos + folio_size(folio); + isize = i_size_read(inode); + verity_in_progress = f2fs_verity_in_progress(inode); + poff = 0; + end_index = 0; + + if (retry) { + if (unlikely(folio->mapping != mapping)) + goto retry_out; + + if (!folio_test_dirty(folio)) + goto retry_out; + + if (folio_test_writeback(folio)) { + if (wbc->sync_mode == WB_SYNC_NONE) + goto retry_out; + f2fs_folio_wait_writeback(folio, DATA, true, true); + } + + if (!folio_clear_dirty_for_io(folio)) + goto retry_out; + } + + /* To avoid dealing with the complexity for one subrange is in bio + * while we trylock_op failed before writing another subrange. + * Try to lock_op before any subrange write for the folio. + */ + if (!op_locked) { + if (!f2fs_trylock_op(sbi, &lc)) { + folio_redirty_for_writepage(wbc, folio); + err = 0; + if (wbc->sync_mode != WB_SYNC_ALL) + goto retry_out; + + retry = true; + folio_unlock(folio); + f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + folio_lock(folio); + goto retry; + } + op_locked = true; + } + + if (!verity_in_progress) { + poff = offset_in_folio(folio, isize); + end_index = isize >> PAGE_SHIFT; + + if (folio->index > end_index || + (folio->index == end_index && poff == 0)) + goto out; + + if (end_pos > isize) { + folio_zero_segment(folio, poff, folio_size(folio)); + end_pos = isize; + } + } + + folio_start_writeback(folio); + + if (folio_test_large(folio)) { + ffs = ffs_find_or_alloc(folio); + ffs_mark_subrange_dirty(folio, 0, end_pos - pos); + if (!bias_added) { + /* Add a bias to prevent premature folio_end_writeabck() */ + WARN_ON_ONCE(atomic_read(&ffs->write_pages_pending) != 0); + atomic_inc(&ffs->write_pages_pending); + bias_added = true; + } + } + + while ((r_len = ffs_find_dirty_range(folio, &pos, end_pos))) { + err = f2fs_write_single_data_folio(folio, &submitted, + wbc, io_type, pos, pos + r_len); + folio_submitted += submitted; + if (err) + goto out; + + nwritten += submitted; + pos += r_len; + } + + if (!err && folio_submitted && + f2fs_is_atomic_file(inode) && + folio_test_f2fs_atomic(folio)) + folio_clear_f2fs_atomic(folio); + +out: + ffs_clear_subrange_dirty(folio, 0, folio_size(folio)); + inode_dec_dirty_pages(inode); + + if (bias_added) { + if (atomic_dec_and_test(&ffs->write_pages_pending)) + folio_end_writeback(folio); + } else if (!folio_submitted && folio_test_writeback(folio)) { + folio_end_writeback(folio); + } + +retry_out: + if (folio_test_locked(folio)) + folio_unlock(folio); + + if (op_locked) { + f2fs_unlock_op(sbi, &lc); + op_locked = false; + } + + if (err || !next) + break; + } + + if (nwritten) + f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, + NULL, 0, DATA); + + return err; +} + static inline bool __should_serialize_io(struct inode *inode, struct writeback_control *wbc) { @@ -3994,7 +4379,10 @@ static int __f2fs_write_data_pages(struct address_space *mapping, account_writeback(inode, true); blk_start_plug(&plug); - ret = f2fs_write_cache_pages(mapping, wbc, io_type); + if (mapping_large_folio_support(inode->i_mapping)) + ret = f2fs_write_cache_folios(mapping, wbc, io_type); + else + ret = f2fs_write_cache_pages(mapping, wbc, io_type); blk_finish_plug(&plug); account_writeback(inode, false); -- 2.34.1 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
