Hi Xiang, On Mon, 13 Dec 2021 09:40:39 +0800 Gao Xiang <[email protected]> wrote:
> Hi Yue, > > On Sun, Dec 12, 2021 at 06:31:28PM +0800, Yue Hu wrote: > > Add tail-packing inline compressed data support for erofsfuse. > > > > Signed-off-by: Yue Hu <[email protected]> > > --- > > v4: > > - introduce EROFS_GET_BLOCKS_FINDTAIL suggested by Xiang. > > - remove 3 functions about calculation to inline data address > > and calculate it directly when reload index. > > - add m_nxtioff/z_idataoff to help get/record the inline data address. > > - add on-disk feature related. > > > > v3: > > - remove z_idata_addr, add z_idata_headlcn instead of m_taillcn. > > - add bug_on for legacy if enable inline and disable big pcluster. > > - extract z_erofs_do_map_blocks() instead of added > > z_erofs_map_tail_data_blocks() with similar logic. > > > > v2: > > - add tail-packing information to inode and get it on first read. > > - update tail-packing checking logic. > > > > include/erofs/internal.h | 8 ++++ > > include/erofs_fs.h | 10 +++- > > lib/decompress.c | 2 +- > > lib/namei.c | 2 +- > > lib/zmap.c | 99 ++++++++++++++++++++++++++++++++-------- > > 5 files changed, 98 insertions(+), 23 deletions(-) > > > > diff --git a/include/erofs/internal.h b/include/erofs/internal.h > > index 8b154ed..1c92a27 100644 > > --- a/include/erofs/internal.h > > +++ b/include/erofs/internal.h > > @@ -110,6 +110,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, > > INCOMPAT_LZ4_0PADDING) > > EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) > > EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) > > EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) > > +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) > > EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) > > > > #define EROFS_I_EA_INITED (1 << 0) > > @@ -171,6 +172,9 @@ struct erofs_inode { > > uint8_t z_algorithmtype[2]; > > uint8_t z_logical_clusterbits; > > uint8_t z_physical_clusterblks; > > + uint16_t z_idata_size; > > + uint32_t z_idata_headlcn; > > + uint64_t z_idataoff; > > }; > > }; > > #ifdef WITH_ANDROID > > @@ -257,8 +261,12 @@ struct erofs_map_blocks { > > > > unsigned int m_flags; > > erofs_blk_t index; > > + > > + u16 m_nxtioff; > > better rename as nextpackoff and move into map_recorder. agree. > > > }; > > > > +#define EROFS_GET_BLOCKS_FINDTAIL 0x0001 > > + > > /* super.c */ > > int erofs_read_superblock(void); > > > > diff --git a/include/erofs_fs.h b/include/erofs_fs.h > > index 66a68e3..0e87e85 100644 > > --- a/include/erofs_fs.h > > +++ b/include/erofs_fs.h > > @@ -22,11 +22,13 @@ > > #define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 > > #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 > > #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 > > +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 > > #define EROFS_ALL_FEATURE_INCOMPAT \ > > (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ > > EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ > > EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ > > - EROFS_FEATURE_INCOMPAT_CHUNKED_FILE) > > + EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ > > + EROFS_FEATURE_INCOMPAT_ZTAILPACKING) > > > > #define EROFS_SB_EXTSLOT_SIZE 16 > > > > @@ -266,13 +268,17 @@ struct z_erofs_lz4_cfgs { > > * (4B) + 2B + (4B) if compacted 2B is on. > > * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) > > * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) > > + * bit 3 : tailpacking inline data > > */ > > #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 > > #define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 > > #define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 > > +#define Z_EROFS_ADVISE_INLINE_DATA 0x0008 > > > > struct z_erofs_map_header { > > - __le32 h_reserved1; > > + __le16 h_reserved1; > > + /* record the size of tailpacking data */ > > + __le16 h_idata_size; > > __le16 h_advise; > > /* > > * bit 0-3 : algorithm type of head 1 (logical cluster type 01); > > diff --git a/lib/decompress.c b/lib/decompress.c > > index 2ee1439..9b90d18 100644 > > --- a/lib/decompress.c > > +++ b/lib/decompress.c > > @@ -67,7 +67,7 @@ out: > > int z_erofs_decompress(struct z_erofs_decompress_req *rq) > > { > > if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) { > > - if (rq->inputsize != EROFS_BLKSIZ) > > + if (rq->inputsize > EROFS_BLKSIZ) > > return -EFSCORRUPTED; > > > > DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ); > > diff --git a/lib/namei.c b/lib/namei.c > > index b4bdabf..481b33e 100644 > > --- a/lib/namei.c > > +++ b/lib/namei.c > > @@ -137,7 +137,7 @@ static int erofs_read_inode_from_disk(struct > > erofs_inode *vi) > > vi->u.chunkbits = LOG_BLOCK_SIZE + > > (vi->u.chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); > > } else if (erofs_inode_is_data_compressed(vi->datalayout)) > > - z_erofs_fill_inode(vi); > > + return z_erofs_fill_inode(vi); > > return 0; > > bogusimode: > > erofs_err("bogus i_mode (%o) @ nid %llu", vi->i_mode, vi->nid | 0ULL); > > diff --git a/lib/zmap.c b/lib/zmap.c > > index 458030b..aa51b61 100644 > > --- a/lib/zmap.c > > +++ b/lib/zmap.c > > @@ -10,6 +10,10 @@ > > #include "erofs/io.h" > > #include "erofs/print.h" > > > > +static int z_erofs_do_map_blocks(struct erofs_inode *vi, > > + struct erofs_map_blocks *map, > > + int flags); > > + > > int z_erofs_fill_inode(struct erofs_inode *vi) > > { > > if (!erofs_sb_has_big_pcluster() && > > @@ -18,8 +22,13 @@ int z_erofs_fill_inode(struct erofs_inode *vi) > > vi->z_algorithmtype[0] = 0; > > vi->z_algorithmtype[1] = 0; > > vi->z_logical_clusterbits = LOG_BLOCK_SIZE; > > + vi->z_idata_size = 0; > > > > vi->flags |= EROFS_I_Z_INITED; > > + if (erofs_sb_has_ztailpacking()) { > > + erofs_err("unsupported, plz enable big pcluster for > > legacy compression inline"); > > + return -EOPNOTSUPP; > > + } > > } > > return 0; > > } > > @@ -44,6 +53,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi) > > > > h = (struct z_erofs_map_header *)buf; > > vi->z_advise = le16_to_cpu(h->h_advise); > > + vi->z_idata_size = le16_to_cpu(h->h_idata_size); > > vi->z_algorithmtype[0] = h->h_algorithmtype & 15; > > vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; > > > > @@ -61,6 +71,16 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode > > *vi) > > vi->nid * 1ULL); > > return -EFSCORRUPTED; > > } > > + > > + if (vi->z_idata_size) { > > + struct erofs_map_blocks map = { .index = UINT_MAX }; > > + > > + ret = z_erofs_do_map_blocks(vi, &map, > > + EROFS_GET_BLOCKS_FINDTAIL); > > + if (ret) > > + return ret; > > + } > > + > > vi->flags |= EROFS_I_Z_INITED; > > return 0; > > } > > @@ -113,6 +133,8 @@ static int legacy_load_cluster_from_disk(struct > > z_erofs_maprecorder *m, > > err = z_erofs_reload_indexes(m, erofs_blknr(pos)); > > if (err) > > return err; > > + m->map->m_nxtioff = erofs_blkoff(pos) + > > + sizeof(struct z_erofs_vle_decompressed_index); > > m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); rt. > > > > > m->lcn = lcn; > > di = m->kaddr + erofs_blkoff(pos); > > @@ -285,7 +307,9 @@ static int compacted_load_cluster_from_disk(struct > > z_erofs_maprecorder *m, > > if (compacted_4b_initial == 32 / 4) > > compacted_4b_initial = 0; > > > > - if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) > > + if (compacted_4b_initial > totalidx) > > + compacted_4b_initial = compacted_2b = 0; > > + else if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) > > compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); > > else > > compacted_2b = 0; > > @@ -310,6 +334,9 @@ out: > > err = z_erofs_reload_indexes(m, erofs_blknr(pos)); > > if (err) > > return err; > > + m->map->m_nxtioff = erofs_blkoff(pos) + > > + ((1 << amortizedshift == 2) ? 2 : ((lcn % 2) ? 4 : 8)); > > > > I'd like to skip the whole current pack for compact indexes. > > How about this: > > static int unpack_compacted_index(struct z_erofs_maprecorder *m, > unsigned int amortizedshift, > - unsigned int eofs, bool lookahead) > - unsigned int eofs) > ++ unsigned int pos, bool lookahead) > { > struct erofs_inode *const vi = m->inode; > const unsigned int lclusterbits = vi->z_logical_clusterbits; > const unsigned int lomask = (1 << lclusterbits) - 1; > -- unsigned int vcnt, base, lo, encodebits, nblk; > ++ unsigned int vcnt, base, lo, encodebits, nblk, eofs; > int i; > u8 *in, type; > bool big_pcluster; > @@@ -203,8 -200,8 +225,10 @@@ > else > return -EOPNOTSUPP; > > ++ m->nextpackoff = round_up(pos, vcnt << amortizedshift); looks more simple, let me check. > big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; > encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; > ++ eofs = erofs_blkoff(pos); > base = round_down(eofs, vcnt << amortizedshift); > in = m->kaddr + base; > > @@@ -340,8 -334,10 +366,7 @@@ out > err = z_erofs_reload_indexes(m, erofs_blknr(pos)); > if (err) > return err; > - return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos), > - lookahead); > - m->map->m_nxtioff = erofs_blkoff(pos) + > - ((1 << amortizedshift == 2) ? 2 : ((lcn % 2) ? 4 : > 8)); > - > - return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos)); > ++ return unpack_compacted_index(m, amortizedshift, pos, lookahead); > } > > Thanks, > Gao Xiang > Thanks.
