This patch introduces a --worker=# parameter for the incoming multi-threaded compression support. It also introduces a segment size used in multi-threaded compression, which has the default value 16MB and cannot be modified.
It also introduces a concept called `segment size` to split large files for multi-threading, which has the default value 16MB for now. Signed-off-by: Yifan Zhao <[email protected]> --- include/erofs/config.h | 4 ++++ lib/config.c | 4 ++++ mkfs/main.c | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/include/erofs/config.h b/include/erofs/config.h index 73e3ac2..d2f91ff 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -75,6 +75,10 @@ struct erofs_configure { char c_force_chunkformat; /* < 0, xattr disabled and INT_MAX, always use inline xattrs */ int c_inline_xattr_tolerance; +#ifdef EROFS_MT_ENABLED + u64 c_segment_size; + u32 c_mt_workers; +#endif u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed; u32 c_max_decompressed_extent_bytes; diff --git a/lib/config.c b/lib/config.c index 947a183..2530274 100644 --- a/lib/config.c +++ b/lib/config.c @@ -38,6 +38,10 @@ void erofs_init_configure(void) cfg.c_pclusterblks_max = 1; cfg.c_pclusterblks_def = 1; cfg.c_max_decompressed_extent_bytes = -1; +#ifdef EROFS_MT_ENABLED + cfg.c_segment_size = 16ULL * 1024 * 1024; + cfg.c_mt_workers = 1; +#endif erofs_stdout_tty = isatty(STDOUT_FILENO); } diff --git a/mkfs/main.c b/mkfs/main.c index 7aea64a..c5cdaf9 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -73,6 +73,9 @@ static struct option long_options[] = { {"gzip", no_argument, NULL, 517}, #endif {"offset", required_argument, NULL, 518}, +#ifdef EROFS_MT_ENABLED + {"workers", required_argument, NULL, 519}, +#endif {0, 0, 0, 0}, }; @@ -175,6 +178,9 @@ static void usage(int argc, char **argv) " --product-out=X X=product_out directory\n" " --fs-config-file=X X=fs_config file\n" " --block-list-file=X X=block_list file\n" +#endif +#ifdef EROFS_MT_ENABLED + " --workers=# set the number of worker threads to # (default=1)\n" #endif ); } @@ -404,6 +410,13 @@ static void erofs_rebuild_cleanup(void) rebuild_src_count = 0; } +#ifdef EROFS_MT_ENABLED +static u32 mkfs_max_worker_num() { + u32 ncpu = erofs_get_available_processors(); + return ncpu ? ncpu : 16; +} +#endif + static int mkfs_parse_options_cfg(int argc, char *argv[]) { char *endptr; @@ -642,6 +655,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) return -EINVAL; } break; +#ifdef EROFS_MT_ENABLED + case 519: + cfg.c_mt_workers = strtoul(optarg, &endptr, 0); + if (errno || *endptr != '\0') { + erofs_err("invalid worker number %s", optarg); + return -EINVAL; + } + if (cfg.c_mt_workers > mkfs_max_worker_num()) { + erofs_warn( + "worker number %s is too large, setting to %ud", + optarg, mkfs_max_worker_num()); + cfg.c_mt_workers = mkfs_max_worker_num(); + } + break; +#endif case 'V': version(); exit(0); @@ -784,6 +812,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) } cfg.c_pclusterblks_packed = pclustersize_packed >> sbi.blkszbits; } + +#ifdef EROFS_MT_ENABLED + if (cfg.c_mt_workers > 1 && + (cfg.c_dedupe || cfg.c_fragments || cfg.c_ztailpacking)) { + cfg.c_mt_workers = 1; + erofs_warn("Please note that dedupe/fragments/ztailpacking" + "is NOT supported in multi-threaded mode now, using worker=1."); + } +#endif + return 0; } -- 2.44.0
