Currently, we have already support tail-packing inline for uncompressed file, let's also support it for compressed file to decrease tail extent I/O and save more space.
Signed-off-by: Yue Hu <[email protected]> --- fs/erofs/compress.h | 2 +- fs/erofs/decompressor.c | 29 ++++--- fs/erofs/erofs_fs.h | 10 ++- fs/erofs/internal.h | 3 + fs/erofs/super.c | 3 + fs/erofs/zdata.c | 76 +++++++++++++++---- fs/erofs/zdata.h | 7 +- fs/erofs/zmap.c | 164 ++++++++++++++++++++++++++++++++++------ 8 files changed, 238 insertions(+), 56 deletions(-) diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h index 579406504919..9de40229be14 100644 --- a/fs/erofs/compress.h +++ b/fs/erofs/compress.h @@ -12,7 +12,7 @@ struct z_erofs_decompress_req { struct super_block *sb; struct page **in, **out; - unsigned short pageofs_out; + unsigned short pageofs_in, pageofs_out; unsigned int inputsize, outputsize; /* indicate the algorithm will be used for decompression */ diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index bf37fc76b182..9ce4b2d23e29 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -186,7 +186,7 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_decompress_req *rq, u8 *out) { unsigned int inputmargin; - u8 *headpage, *src; + u8 *headpage, *src, *in; bool support_0padding; int ret, maptype; @@ -196,7 +196,7 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_decompress_req *rq, support_0padding = false; /* decompression inplace is only safe when 0padding is enabled */ - if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) { + if (!rq->pageofs_in && erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) { support_0padding = true; while (!headpage[inputmargin & ~PAGE_MASK]) @@ -215,20 +215,22 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_decompress_req *rq, if (IS_ERR(src)) return PTR_ERR(src); + in = src + rq->pageofs_in + inputmargin; + /* legacy format could compress extra data in a pcluster. */ if (rq->partial_decoding || !support_0padding) - ret = LZ4_decompress_safe_partial(src + inputmargin, out, - rq->inputsize, rq->outputsize, rq->outputsize); + ret = LZ4_decompress_safe_partial(in, out, rq->inputsize, + rq->outputsize, rq->outputsize); else - ret = LZ4_decompress_safe(src + inputmargin, out, - rq->inputsize, rq->outputsize); + ret = LZ4_decompress_safe(in, out, rq->inputsize, + rq->outputsize); if (ret != rq->outputsize) { erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", ret, rq->inputsize, inputmargin, rq->outputsize); print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET, - 16, 1, src + inputmargin, rq->inputsize, true); + 16, 1, in, rq->inputsize, true); print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET, 16, 1, out, rq->outputsize, true); @@ -299,7 +301,7 @@ static int z_erofs_shifted_transform(struct z_erofs_decompress_req *rq, { const unsigned int nrpages_out = PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; - const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out; + unsigned int righthalf = PAGE_SIZE - rq->pageofs_out; unsigned char *src, *dst; if (nrpages_out > 2) { @@ -312,20 +314,25 @@ static int z_erofs_shifted_transform(struct z_erofs_decompress_req *rq, return 0; } + if (nrpages_out == 1 && rq->outputsize < righthalf) + righthalf = rq->outputsize; + src = kmap_atomic(*rq->in); if (rq->out[0]) { dst = kmap_atomic(rq->out[0]); - memcpy(dst + rq->pageofs_out, src, righthalf); + memcpy(dst + rq->pageofs_out, src + rq->pageofs_in, righthalf); kunmap_atomic(dst); } if (nrpages_out == 2) { DBG_BUGON(!rq->out[1]); if (rq->out[1] == *rq->in) { - memmove(src, src + righthalf, rq->pageofs_out); + memmove(src, src + rq->pageofs_in + righthalf, + rq->pageofs_out); } else { dst = kmap_atomic(rq->out[1]); - memcpy(dst, src + righthalf, rq->pageofs_out); + memcpy(dst, src + rq->pageofs_in + righthalf, + rq->pageofs_out); kunmap_atomic(dst); } } diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 083997a034e5..19153cd6f5e8 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -23,13 +23,15 @@ #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 #define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_TAIL_PACKING 0x00000010 #define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ - EROFS_FEATURE_INCOMPAT_COMPR_HEAD2) + EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \ + EROFS_FEATURE_INCOMPAT_TAIL_PACKING) #define EROFS_SB_EXTSLOT_SIZE 16 @@ -292,13 +294,17 @@ struct z_erofs_lzma_cfgs { * (4B) + 2B + (4B) if compacted 2B is on. * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) + * bit 3 : tail-packing inline data */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_DATA 0x0008 struct z_erofs_map_header { - __le32 h_reserved1; + __le16 h_reserved1; + /* record the size of tail-packing pcluster */ + __le16 h_idata_size; __le16 h_advise; /* * bit 0-3 : algorithm type of head 1 (logical cluster type 01); diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 3265688af7f9..9ab3d38c7919 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -262,6 +262,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(tail_packing, incompat, INCOMPAT_TAIL_PACKING) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) /* atomic flag definitions */ @@ -296,6 +297,8 @@ struct erofs_inode { unsigned short z_advise; unsigned char z_algorithmtype[2]; unsigned char z_logical_clusterbits; + unsigned short z_idata_size; + unsigned long z_idata_headlcn; }; #endif /* CONFIG_EROFS_FS_ZIP */ }; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 6a969b1e0ee6..6f8bb83795a1 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -411,6 +411,9 @@ static int erofs_read_superblock(struct super_block *sb) /* handle multiple devices */ ret = erofs_init_devices(sb, dsb); + + if (erofs_sb_has_tail_packing(sbi)) + erofs_info(sb, "EXPERIMENTAL compression inline data feature in use. Use at your own risk!"); out: kunmap(page); put_page(page); diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index bcb1b91b234f..85c46170598d 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -432,7 +432,7 @@ static int z_erofs_lookup_collection(struct z_erofs_collector *clt, } cl = z_erofs_primarycollection(pcl); - if (cl->pageofs != (map->m_la & ~PAGE_MASK)) { + if (cl->pageofs_out != (map->m_la & ~PAGE_MASK)) { DBG_BUGON(1); return -EFSCORRUPTED; } @@ -472,6 +472,7 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, struct z_erofs_pcluster *pcl; struct z_erofs_collection *cl; struct erofs_workgroup *grp; + unsigned int nrpages; int err; if (!(map->m_flags & EROFS_MAP_ENCODED)) { @@ -479,13 +480,23 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, return -EFSCORRUPTED; } + if (map->m_plen < EROFS_BLKSIZ && !(map->m_flags & EROFS_MAP_META)) + nrpages = 1; + else + nrpages = map->m_plen >> PAGE_SHIFT; + /* no available pcluster, let's allocate one */ - pcl = z_erofs_alloc_pcluster(map->m_plen >> PAGE_SHIFT); + pcl = z_erofs_alloc_pcluster(nrpages); if (IS_ERR(pcl)) return PTR_ERR(pcl); - atomic_set(&pcl->obj.refcount, 1); - pcl->obj.index = map->m_pa >> PAGE_SHIFT; + pcl->pclusterlen = map->m_plen < EROFS_BLKSIZ ? map->m_plen : 0; + + if (!(map->m_flags & EROFS_MAP_META)) { + atomic_set(&pcl->obj.refcount, 1); + pcl->obj.index = map->m_pa >> PAGE_SHIFT; + } + pcl->algorithmformat = map->m_algorithmformat; pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) | (map->m_flags & EROFS_MAP_FULL_MAPPED ? @@ -496,7 +507,16 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, clt->mode = COLLECT_PRIMARY_FOLLOWED; cl = z_erofs_primarycollection(pcl); - cl->pageofs = map->m_la & ~PAGE_MASK; + cl->pageofs_out = map->m_la & ~PAGE_MASK; + cl->pageofs_in = 0; + + /* no inplace I/O if inline and fill compressed page first */ + if (map->m_flags & EROFS_MAP_META) { + cl->pageofs_in = map->m_pa & ~PAGE_MASK; + + clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE; + WRITE_ONCE(pcl->compressed_pages[0], map->mpage); + } /* * lock all primary followed works before visible to others @@ -505,6 +525,9 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, mutex_init(&cl->lock); DBG_BUGON(!mutex_trylock(&cl->lock)); + if (map->m_flags & EROFS_MAP_META) + goto skip_workgroup; + grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj); if (IS_ERR(grp)) { err = PTR_ERR(grp); @@ -516,6 +539,8 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, err = -EEXIST; goto err_out; } + +skip_workgroup: /* used to check tail merging loop due to corrupted images */ if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL) clt->tailpcl = pcl; @@ -543,6 +568,9 @@ static int z_erofs_collector_begin(struct z_erofs_collector *clt, DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL); DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + if (map->m_flags & EROFS_MAP_META) + goto collection; + if (!PAGE_ALIGNED(map->m_pa)) { DBG_BUGON(1); return -EINVAL; @@ -552,6 +580,7 @@ static int z_erofs_collector_begin(struct z_erofs_collector *clt, if (grp) { clt->pcl = container_of(grp, struct z_erofs_pcluster, obj); } else { +collection: ret = z_erofs_register_collection(clt, inode, map); if (!ret) @@ -692,6 +721,9 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, if (err) goto err_out; + if (map->m_flags & EROFS_MAP_META) + goto hitted; + /* preload all compressed pages (maybe downgrade role if necessary) */ if (should_alloc_managed_pages(fe, sbi->opt.cache_strategy, map->m_la)) cache_strategy = TRYALLOC; @@ -832,6 +864,13 @@ static void z_erofs_decompressqueue_endio(struct bio *bio) bio_put(bio); } +static inline bool z_erofs_pcluster_is_inline(struct z_erofs_pcluster *pcl) +{ + struct z_erofs_collection *cl = z_erofs_primarycollection(pcl); + + return !!cl->pageofs_in; +} + static int z_erofs_decompress_pcluster(struct super_block *sb, struct z_erofs_pcluster *pcl, struct page **pagepool) @@ -963,20 +1002,22 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, goto out; llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT; - if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) { + if (nr_pages << PAGE_SHIFT >= cl->pageofs_out + llen) { outputsize = llen; partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH); } else { - outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs; + outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs_out; partial = true; } - inputsize = pcl->pclusterpages * PAGE_SIZE; + inputsize = pcl->pclusterlen ? pcl->pclusterlen : + pcl->pclusterpages * PAGE_SIZE; err = z_erofs_decompress(&(struct z_erofs_decompress_req) { .sb = sb, .in = compressed_pages, .out = pages, - .pageofs_out = cl->pageofs, + .pageofs_in = cl->pageofs_in, + .pageofs_out = cl->pageofs_out, .inputsize = inputsize, .outputsize = outputsize, .alg = pcl->algorithmformat, @@ -1290,6 +1331,13 @@ static void z_erofs_submit_queue(struct super_block *sb, pcl = container_of(owned_head, struct z_erofs_pcluster, next); + /* close the main owned chain at first */ + owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, + Z_EROFS_PCLUSTER_TAIL_CLOSED); + + if (z_erofs_pcluster_is_inline(pcl)) + goto noio_submission; + /* no device id here, thus it will always succeed */ mdev = (struct erofs_map_dev) { .m_pa = blknr_to_addr(pcl->obj.index), @@ -1299,10 +1347,6 @@ static void z_erofs_submit_queue(struct super_block *sb, cur = erofs_blknr(mdev.m_pa); end = cur + pcl->pclusterpages; - /* close the main owned chain at first */ - owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, - Z_EROFS_PCLUSTER_TAIL_CLOSED); - do { struct page *page; @@ -1341,10 +1385,12 @@ static void z_erofs_submit_queue(struct super_block *sb, bypass = false; } while (++cur < end); - if (!bypass) + if (!bypass) { qtail[JQ_SUBMIT] = &pcl->next; - else + } else { +noio_submission: move_to_bypass_jobqueue(pcl, qtail, owned_head); + } } while (owned_head != Z_EROFS_PCLUSTER_TAIL); if (bio) diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h index 879df5362777..cc724c51eda8 100644 --- a/fs/erofs/zdata.h +++ b/fs/erofs/zdata.h @@ -25,8 +25,8 @@ struct z_erofs_collection { struct mutex lock; - /* I: page offset of start position of decompression */ - unsigned short pageofs; + /* I: page offset of start position of (de)compression */ + unsigned short pageofs_in, pageofs_out; /* L: maximum relative page index in pagevec[] */ unsigned short nr_pages; @@ -62,6 +62,9 @@ struct z_erofs_pcluster { /* A: lower limit of decompressed length and if full length or not */ unsigned int length; + /* I: tail-packing physical cluster length */ + unsigned short pclusterlen; + /* I: physical cluster size in pages */ unsigned short pclusterpages; diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 660489a7fb64..e62b81c94865 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -7,6 +7,10 @@ #include <asm/unaligned.h> #include <trace/events/erofs.h> +static int z_erofs_do_map_blocks(struct inode *inode, + struct erofs_map_blocks *map, + int flags); + int z_erofs_fill_inode(struct inode *inode) { struct erofs_inode *const vi = EROFS_I(inode); @@ -18,12 +22,74 @@ int z_erofs_fill_inode(struct inode *inode) vi->z_algorithmtype[0] = 0; vi->z_algorithmtype[1] = 0; vi->z_logical_clusterbits = LOG_BLOCK_SIZE; + vi->z_idata_size = 0; set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); + + DBG_BUGON(erofs_sb_has_tail_packing(sbi)); } inode->i_mapping->a_ops = &z_erofs_aops; return 0; } +static erofs_off_t compacted_inline_data_addr(struct erofs_inode *vi, + erofs_off_t i_off, + unsigned int totalidx) +{ + const erofs_off_t ebase = ALIGN(i_off + vi->inode_isize + + vi->xattr_isize, 8) + + sizeof(struct z_erofs_map_header); + unsigned int compacted_4b_initial, compacted_4b_end; + unsigned int compacted_2b; + erofs_off_t addr; + + compacted_4b_initial = (32 - ebase % 32) / 4; + if (compacted_4b_initial == 32 / 4) + compacted_4b_initial = 0; + + if (compacted_4b_initial > totalidx) { + compacted_4b_initial = 0; + compacted_2b = 0; + } else if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) { + compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); + } else + compacted_2b = 0; + + compacted_4b_end = totalidx - compacted_4b_initial - compacted_2b; + + addr = ebase + compacted_4b_initial * 4 + compacted_2b * 2; + if (compacted_4b_end > 1) + addr += (compacted_4b_end / 2) * 8; + if (compacted_4b_end % 2) + addr += 8; + + return addr; +} + +static erofs_off_t legacy_inline_data_addr(struct erofs_inode *vi, + erofs_off_t i_off, + unsigned int totalidx) +{ + return Z_EROFS_VLE_LEGACY_INDEX_ALIGN(i_off + vi->inode_isize + + vi->xattr_isize) + + totalidx * sizeof(struct z_erofs_vle_decompressed_index); +} + +static erofs_off_t z_erofs_inline_data_addr(struct inode *inode) +{ + struct erofs_inode *const vi = EROFS_I(inode); + const unsigned int datamode = vi->datalayout; + const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); + const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid); + + if (datamode == EROFS_INODE_FLAT_COMPRESSION) + return compacted_inline_data_addr(vi, ibase, totalidx); + + if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) + return legacy_inline_data_addr(vi, ibase, totalidx); + + return -EINVAL; +} + static int z_erofs_fill_inode_lazy(struct inode *inode) { struct erofs_inode *const vi = EROFS_I(inode); @@ -65,6 +131,7 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) h = kaddr + erofs_blkoff(pos); vi->z_advise = le16_to_cpu(h->h_advise); + vi->z_idata_size = le16_to_cpu(h->h_idata_size); vi->z_algorithmtype[0] = h->h_algorithmtype & 15; vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; @@ -94,13 +161,29 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) err = -EFSCORRUPTED; goto unmap_done; } - /* paired with smp_mb() at the beginning of the function */ - smp_mb(); - set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); unmap_done: kunmap_atomic(kaddr); unlock_page(page); put_page(page); + if (err) + goto out_unlock; + + /* record HEAD lcn for tail-packing pcluster */ + if (vi->z_idata_size) { + struct erofs_map_blocks map = { .m_la = inode->i_size - 1 }; + + err = z_erofs_do_map_blocks(inode, &map, 0); + if (map.mpage) + put_page(map.mpage); + if (err < 0) + goto out_unlock; + + vi->z_idata_headlcn = map.m_la >> vi->z_logical_clusterbits; + } + + /* paired with smp_mb() at the beginning of the function */ + smp_mb(); + set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); out_unlock: clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); return err; @@ -305,7 +388,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, } m->clusterofs = lo; m->delta[0] = 0; - /* figout out blkaddr (pblk) for HEAD lclusters */ + /* figure out blkaddr (pblk) for HEAD lclusters */ if (!big_pcluster) { nblk = 1; while (i > 0) { @@ -466,7 +549,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, } static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, - unsigned int initial_lcn) + unsigned int initial_lcn, + bool idatamap) { struct erofs_inode *const vi = EROFS_I(m->inode); struct erofs_map_blocks *const map = m->map; @@ -479,6 +563,11 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2); DBG_BUGON(m->type != m->headtype); + if (idatamap) { + map->m_plen = vi->z_idata_size; + return 0; + } + if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) && !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || @@ -583,9 +672,9 @@ static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) return 0; } -int z_erofs_map_blocks_iter(struct inode *inode, - struct erofs_map_blocks *map, - int flags) +static int z_erofs_do_map_blocks(struct inode *inode, + struct erofs_map_blocks *map, + int flags) { struct erofs_inode *const vi = EROFS_I(inode); struct z_erofs_maprecorder m = { @@ -596,20 +685,7 @@ int z_erofs_map_blocks_iter(struct inode *inode, unsigned int lclusterbits, endoff; unsigned long initial_lcn; unsigned long long ofs, end; - - trace_z_erofs_map_blocks_iter_enter(inode, map, flags); - - /* when trying to read beyond EOF, leave it unmapped */ - if (map->m_la >= inode->i_size) { - map->m_llen = map->m_la + 1 - inode->i_size; - map->m_la = inode->i_size; - map->m_flags = 0; - goto out; - } - - err = z_erofs_fill_inode_lazy(inode); - if (err) - goto out; + bool idatamap; lclusterbits = vi->z_logical_clusterbits; ofs = map->m_la; @@ -658,10 +734,23 @@ int z_erofs_map_blocks_iter(struct inode *inode, goto unmap_out; } + /* check if mapping tail-packing data */ + idatamap = vi->z_idata_size && (ofs == inode->i_size - 1 || + m.lcn == vi->z_idata_headlcn); + + /* need to trim tail-packing data if beyond file size */ map->m_llen = end - map->m_la; - map->m_pa = blknr_to_addr(m.pblk); + if (idatamap && end > inode->i_size) + map->m_llen -= end - inode->i_size; - err = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (idatamap && (vi->z_advise & Z_EROFS_ADVISE_INLINE_DATA)) { + map->m_pa = z_erofs_inline_data_addr(inode); + map->m_flags |= EROFS_MAP_META; + } else { + map->m_pa = blknr_to_addr(m.pblk); + } + + err = z_erofs_get_extent_compressedlen(&m, initial_lcn, idatamap); if (err) goto out; @@ -689,9 +778,34 @@ int z_erofs_map_blocks_iter(struct inode *inode, __func__, map->m_la, map->m_pa, map->m_llen, map->m_plen, map->m_flags); + return err; +} + +int z_erofs_map_blocks_iter(struct inode *inode, + struct erofs_map_blocks *map, + int flags) +{ + int err = 0; + + trace_z_erofs_map_blocks_iter_enter(inode, map, flags); + + /* when trying to read beyond EOF, leave it unmapped */ + if (map->m_la >= inode->i_size) { + map->m_llen = map->m_la + 1 - inode->i_size; + map->m_la = inode->i_size; + map->m_flags = 0; + goto out; + } + + err = z_erofs_fill_inode_lazy(inode); + if (err) + goto out; + + err = z_erofs_do_map_blocks(inode, map, flags); +out: trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); - /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ + /* aggressively BUG_ON if CONFIG_EROFS_FS_DEBUG is on */ DBG_BUGON(err < 0 && err != -ENOMEM); return err; } -- 2.17.1
