Re: [PATCH v2 3/7] erofs-utils: mkfs: add --worker=# parameter

[Gao Xiang]

On 2024/2/20 15:55, Yifan Zhao wrote:

This patch introduces a --worker=# parameter for the incoming
multi-threaded compression support. It also introduces a segment size
used in multi-threaded compression, which has the default value 16MB
and cannot be modified.

It also introduces a concept called `segment size` to split large files
for multi-threading, which has the default value 16MB for now.

Signed-off-by: Yifan Zhao <zhaoyi...@sjtu.edu.cn>
---
  include/erofs/config.h |  4 ++++
  lib/config.c           |  4 ++++
  mkfs/main.c            | 38 ++++++++++++++++++++++++++++++++++++++
  3 files changed, 46 insertions(+)

diff --git a/include/erofs/config.h b/include/erofs/config.h
index 73e3ac2..d19094e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -75,6 +75,10 @@ struct erofs_configure {
        char c_force_chunkformat;
        /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
        int c_inline_xattr_tolerance;
+#ifdef EROFS_MT_ENABLED
+       u64 c_mt_segment_size;

I think mt_ prefix is not needed: c_segment_size;

+       u32 c_mt_worker_num;

c_mt_workers;

+#endif

u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;

        u32 c_max_decompressed_extent_bytes;
diff --git a/lib/config.c b/lib/config.c
index 947a183..8add06d 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -38,6 +38,10 @@ void erofs_init_configure(void)
        cfg.c_pclusterblks_max = 1;
        cfg.c_pclusterblks_def = 1;
        cfg.c_max_decompressed_extent_bytes = -1;
+#ifdef EROFS_MT_ENABLED
+       cfg.c_mt_segment_size = 16ULL * 1024 * 1024;
+       cfg.c_mt_worker_num = 1;
+#endif

erofs_stdout_tty = isatty(STDOUT_FILENO);

  }
diff --git a/mkfs/main.c b/mkfs/main.c
index 7aea64a..3882533 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -73,6 +73,9 @@ static struct option long_options[] = {
        {"gzip", no_argument, NULL, 517},
  #endif
        {"offset", required_argument, NULL, 518},
+#ifdef EROFS_MT_ENABLED
+       {"worker", required_argument, NULL, 519},

let's use `--workers=#` instead of `worker`.

+#endif
        {0, 0, 0, 0},
  };

@@ -175,6 +178,9 @@ static void usage(int argc, char **argv)

                " --product-out=X       X=product_out directory\n"
                " --fs-config-file=X    X=fs_config file\n"
                " --block-list-file=X   X=block_list file\n"
+#endif
+#ifdef EROFS_MT_ENABLED
+               " --worker=#            set the number of worker threads to # 
(default=1)\n"

--workers=#

Thanks,
Gao Xiang