On 2025/5/16 16:26, Bo Liu wrote:
> [外部邮件] 此邮件来源于小米公司外部，请谨慎处理。若对邮件安全性存疑，请将邮件转发给mi...@xiaomi.com进行反馈
>
> This patch introdueces the use of the Intel QAT to decompress compressed
> data in the EROFS filesystem, aiming to improve the decompression speed
> of compressed datea.
>
> We created a 285MiB compressed file and then used the following command to
> create EROFS images with different cluster size.
> # mkfs.erofs -zdeflate,level=9 -C16384
>
> fio command was used to test random read and small random read(~5%) and
> sequential read performance.
> # fio -filename=testfile -bs=4k -rw=read -name=job1
> # fio -filename=testfile -bs=4k -rw=randread -name=job1
> # fio -filename=testfile -bs=4k -rw=randread --io_size=14m -name=job1
>
> Here are some performance numbers for reference:
>
> Processors: Intel(R) Xeon(R) 6766E(144 core)
> Memory: 521 GiB
>
> |-----------------------------------------------------------------------------|
> | | Cluster size | sequential read | randread | small randread(5%)
> |
> |-----------|--------------|-----------------|-----------|--------------------|
> | Intel QAT | 4096 | 538 MiB/s | 112 MiB/s | 20.76 MiB/s
> |
> | Intel QAT | 16384 | 699 MiB/s | 158 MiB/s | 21.02 MiB/s
> |
> | Intel QAT | 65536 | 917 MiB/s | 278 MiB/s | 20.90 MiB/s
> |
> | Intel QAT | 131072 | 1056 MiB/s | 351 MiB/s | 23.36 MiB/s
> |
> | Intel QAT | 262144 | 1145 MiB/s | 431 MiB/s | 26.66 MiB/s
> |
> | deflate | 4096 | 499 MiB/s | 108 MiB/s | 21.50 MiB/s
> |
> | deflate | 16384 | 422 MiB/s | 125 MiB/s | 18.94 MiB/s
> |
> | deflate | 65536 | 452 MiB/s | 159 MiB/s | 13.02 MiB/s
> |
> | deflate | 131072 | 452 MiB/s | 177 MiB/s | 11.44 MiB/s
> |
> | deflate | 262144 | 466 MiB/s | 194 MiB/s | 10.60 MiB/s
> |
>
> Signed-off-by: Bo Liu <liub...@inspur.com>
> ---
> v1:
> https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liub...@inspur.com/
> v2:
> https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liub...@inspur.com/T/#t
>
> Changes since v2:
> - Add a new kernel config option CONFIG_EROFS_FS_ZIP_CRYPTO.
> - Modify the sysfs interface to merge enable and disable into
> a single entry point, and add a read function to view the current state.
> - Optimize the relevant code,use the sg_alloc_table_from_pages_segment
> interface.
>
> fs/erofs/Kconfig | 14 +++
> fs/erofs/Makefile | 1 +
> fs/erofs/compress.h | 14 +++
> fs/erofs/decompressor_crypto.c | 199 ++++++++++++++++++++++++++++++++
> fs/erofs/decompressor_deflate.c | 16 ++-
> fs/erofs/sysfs.c | 36 ++++++
> 6 files changed, 279 insertions(+), 1 deletion(-)
> create mode 100644 fs/erofs/decompressor_crypto.c
>
> diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
> index 331e49cd1b8d..db49ae3c1922 100644
> --- a/fs/erofs/Kconfig
> +++ b/fs/erofs/Kconfig
> @@ -144,6 +144,20 @@ config EROFS_FS_ZIP_ZSTD
>
> If unsure, say N.
>
> +config EROFS_FS_ZIP_CRYPTO
> + bool "EROFS hardware decompression support (crypto interface)"
> + depends on EROFS_FS_ZIP
> + help
> + Saying Y here includes support for reading EROFS file systems
> + containing crypto compressed data. It gives better decompression
> + speed than the software-implemented compression, and it costs
> + lower CPU overhead.
> +
> + Crypto support is an experimental feature for now and so most
> + file systems will be readable without selecting this option.
> +
> + If unsure, say N.
> +
> config EROFS_FS_ONDEMAND
> bool "EROFS fscache-based on-demand read support (deprecated)"
> depends on EROFS_FS
> diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile
> index 4331d53c7109..247b263eb667 100644
> --- a/fs/erofs/Makefile
> +++ b/fs/erofs/Makefile
> @@ -7,5 +7,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o
> zutil.o
> erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
> erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
> erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
> +erofs-$(CONFIG_EROFS_FS_ZIP_CRYPTO) += decompressor_crypto.o
> erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o
> erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o
> diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
> index 2704d7a592a5..356ae2473bd5 100644
> --- a/fs/erofs/compress.h
> +++ b/fs/erofs/compress.h
> @@ -70,10 +70,24 @@ struct z_erofs_stream_dctx {
> bool bounced; /* is the bounce buffer used now? */
> };
>
> +struct z_erofs_crypto_engine {
> + char *crypto_name;
> + bool enabled;
> + struct crypto_acomp *erofs_tfm;
> +};
> +
> +extern struct z_erofs_crypto_engine
> z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2];
> +
> int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
> void **src, struct page **pgpl);
> int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char
> *padbuf,
> unsigned int padbufsize);
> int __init z_erofs_init_decompressor(void);
> void z_erofs_exit_decompressor(void);
> +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp
> *erofs_tfm, struct page **pgpl);
> +struct crypto_acomp *z_erofs_crypto_get_engine(int type);
> +int z_erofs_crypto_enable_engine(const char *name);
> +void z_erofs_crypto_disable_engine(void);
> +int z_erofs_crypto_engine_format(char *buf);
> #endif
> diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
> new file mode 100644
> index 000000000000..55f6a9e6dcf6
> --- /dev/null
> +++ b/fs/erofs/decompressor_crypto.c
> @@ -0,0 +1,199 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +#include <linux/scatterlist.h>
> +#include <crypto/acompress.h>
> +
> +#include "compress.h"
> +
> +static int z_erofs_crypto_decompress_mem(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp *erofs_tfm)
> +{
> + struct sg_table st_src, st_dst;
> + struct acomp_req *req;
> + struct crypto_wait wait;
> + u8 *headpage;
> + int ret;
> +
> + headpage = kmap_local_page(*rq->in);
> + ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
> + min_t(unsigned int, rq->inputsize,
> + rq->sb->s_blocksize -
> rq->pageofs_in));
> + kunmap_local(headpage);
> + if (ret)
> + return ret;
> +
> + req = acomp_request_alloc(erofs_tfm);
> + if (!req) {
> + erofs_err(rq->sb, "failed to alloc decompress request");
> + return -ENOMEM;
> + }
> +
> + ret = sg_alloc_table_from_pages_segment(&st_src,
> + rq->in, rq->inpages, rq->pageofs_in,
> + rq->inputsize, UINT_MAX, GFP_KERNEL);
> + if (ret < 0)
> + goto failed_src_alloc;
> +
> + ret = sg_alloc_table_from_pages_segment(&st_dst,
> + rq->out, rq->outpages,
> rq->pageofs_out,
> + rq->outputsize, UINT_MAX, GFP_KERNEL);
> + if (ret < 0)
> + goto failed_dst_alloc;
> +
> + acomp_request_set_params(req, st_src.sgl,
> + st_dst.sgl, rq->inputsize, rq->outputsize);
> +
> + crypto_init_wait(&wait);
> + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
> + crypto_req_done, &wait);
> +
> + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
> + if (ret < 0) {
> + erofs_err(rq->sb, "failed to decompress %d in[%u, %u]
> out[%u]",
> + ret, rq->inputsize, rq->pageofs_in,
> rq->outputsize);
> + ret = -EIO;
> + } else
> + ret = 0;
> +
> + sg_free_table(&st_dst);
> +failed_dst_alloc:
> + sg_free_table(&st_src);
> +failed_src_alloc:
> + acomp_request_free(req);
> + return ret;
> +}
> +
> +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp *erofs_tfm, struct page
> **pgpl)
> +{
> + int i;
> +
> + for (i = 0; i < rq->outpages; i++) {
> + struct page *const page = rq->out[i];
> + struct page *victim;
> +
> + if (!page) {
> + victim = __erofs_allocpage(pgpl, rq->gfp, true);
> + if (!victim)
> + return -ENOMEM;
> + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
> + rq->out[i] = victim;
> + }
> + }
> +
> + return z_erofs_crypto_decompress_mem(rq, erofs_tfm);
> +}
> +
> +struct crypto_acomp *z_erofs_crypto_get_engine(int type)
> +{
> + int i = 0;
> +
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled)
> + return z_erofs_crypto[type][i].erofs_tfm;
> + i++;
> + }
> + return NULL;
> +}
> +
> +static int z_erofs_crypto_get_compress_type(const char *name)
> +{
> + int i, j;
> +
> + for (i = 0; i < Z_EROFS_COMPRESSION_MAX; i++) {
> + j = 0;
> + while (z_erofs_crypto[i][j].crypto_name) {
Is it possible to use z_erofs_crypto_engine itself instead of crypto_name to
check if there is a supported compression engine, like:
#define Z_EROFS_CRYPTO_ENGINE_MAX 1
struct z_erofs_crypto_engine
*z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][Z_EROFS_CRYPTO_ENGINE_MAX] = {
[Z_EROFS_COMPRESSION_LZ4] = { },
[Z_EROFS_COMPRESSION_LZMA] = { },
[Z_EROFS_COMPRESSION_DEFLATE] = {
&(struct z_erofs_crypto_engine) {
.crypto_name = "qat_deflate",
.enabled = false,
.erofs_tfm = NULL,
}
},
[Z_EROFS_COMPRESSION_ZSTD] = {
},
};
for (i = 0; z_erofs_crypto[type][i]; ++i) {
...
}
> + if (!strcmp(name, z_erofs_crypto[i][j].crypto_name))
> + return i;
> +
> + j++;
> + }
> + }
> + return -EINVAL;
> +}
> +
> +int z_erofs_crypto_enable_engine(const char *name)
> +{
> + int i = 0, type;
> +
> + type = z_erofs_crypto_get_compress_type(name);
> + if (type < 0)
> + return -EINVAL;
> +
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (!strcmp(z_erofs_crypto[type][i].crypto_name, name)) {
> + if (z_erofs_crypto[type][i].enabled)
> + break;
Maybe an error is needed to inform duplicate enable.
> +
> + z_erofs_crypto[type][i].erofs_tfm =
> +
> crypto_alloc_acomp(z_erofs_crypto[type][i].crypto_name, 0, 0);
> + if (IS_ERR(z_erofs_crypto[type][i].erofs_tfm)) {
> + z_erofs_crypto[type][i].erofs_tfm = NULL;
> + return -ENXIO;
> + }
> + z_erofs_crypto[type][i].enabled = true;
> + } else if (z_erofs_crypto[type][i].enabled) {
> + if (z_erofs_crypto[type][i].erofs_tfm)
> +
> crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
> + z_erofs_crypto[type][i].enabled = false;
This may be concurrent with z_erofs_deflate_decompress, so it should be
disabled first and may require a memory barrier.
> + }
> + i++;
> + }
> + return 0;
> +}
> +
> +void z_erofs_crypto_disable_engine(void)
> +{
> + int i = 0, type;
> +
> + for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
> + i = 0;
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled &&
> + z_erofs_crypto[type][i].erofs_tfm) {
> +
> crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
> + z_erofs_crypto[type][i].erofs_tfm = NULL;
> + z_erofs_crypto[type][i].enabled = false;
> + }
> + i++;
> + }
> + }
> +}
> +
> +int z_erofs_crypto_engine_format(char *buf)
> +{
> + int type, i, len = 0;
> +
> + for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
> + i = 0;
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled)
> + len += scnprintf(buf + len, PATH_MAX - len,
> "%s ",
> +
> z_erofs_crypto[type][i].crypto_name);
> + i++;
> + }
> + }
> + return len;
> +}
> +
> +struct z_erofs_crypto_engine z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2] = {
> + [Z_EROFS_COMPRESSION_LZ4] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_LZMA] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_DEFLATE] = {
> + (struct z_erofs_crypto_engine) {
> + .crypto_name = "qat_deflate",
> + .enabled = false,
> + .erofs_tfm = NULL,
> + },
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_ZSTD] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> +};
> diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c
> index c6908a487054..bebbf1eccd3d 100644
> --- a/fs/erofs/decompressor_deflate.c
> +++ b/fs/erofs/decompressor_deflate.c
> @@ -97,7 +97,7 @@ static int z_erofs_load_deflate_config(struct super_block
> *sb,
> return -ENOMEM;
> }
>
> -static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> +static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> struct page **pgpl)
> {
> struct super_block *sb = rq->sb;
> @@ -178,6 +178,20 @@ static int z_erofs_deflate_decompress(struct
> z_erofs_decompress_req *rq,
> return err;
> }
>
> +static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> + struct page **pgpl)
> +{
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + struct crypto_acomp *erofs_tfm = NULL;
> +
> + erofs_tfm = z_erofs_crypto_get_engine(Z_EROFS_COMPRESSION_DEFLATE);
> + if (erofs_tfm && !rq->partial_decoding)
> + return z_erofs_crypto_decompress(rq, erofs_tfm, pgpl);
> + else
Unnecessary else.
> +#endif
> + return __z_erofs_deflate_decompress(rq, pgpl);
> +}
> +
> const struct z_erofs_decompressor z_erofs_deflate_decomp = {
> .config = z_erofs_load_deflate_config,
> .decompress = z_erofs_deflate_decompress,
> diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
> index dad4e6c6c155..00123c4d3196 100644
> --- a/fs/erofs/sysfs.c
> +++ b/fs/erofs/sysfs.c
> @@ -7,12 +7,14 @@
> #include <linux/kobject.h>
>
> #include "internal.h"
> +#include "compress.h"
>
> enum {
> attr_feature,
> attr_drop_caches,
> attr_pointer_ui,
> attr_pointer_bool,
> + attr_crypto,
> };
>
> enum {
> @@ -60,6 +62,9 @@ static struct erofs_attr erofs_attr_##_name = {
> \
> EROFS_ATTR_RW_UI(sync_decompress, erofs_mount_opts);
> EROFS_ATTR_FUNC(drop_caches, 0200);
> #endif
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> +EROFS_ATTR_FUNC(crypto, 0644);
> +#endif
>
> static struct attribute *erofs_attrs[] = {
> #ifdef CONFIG_EROFS_FS_ZIP
> @@ -95,6 +100,9 @@ static struct attribute *erofs_feat_attrs[] = {
> ATTR_LIST(fragments),
> ATTR_LIST(dedupe),
> ATTR_LIST(48bit),
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + ATTR_LIST(crypto),
> +#endif
> NULL,
> };
> ATTRIBUTE_GROUPS(erofs_feat);
> @@ -128,6 +136,10 @@ static ssize_t erofs_attr_show(struct kobject *kobj,
> if (!ptr)
> return 0;
> return sysfs_emit(buf, "%d\n", *(bool *)ptr);
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + case attr_crypto:
> + return z_erofs_crypto_engine_format(buf);
> +#endif
> }
> return 0;
> }
> @@ -141,6 +153,10 @@ static ssize_t erofs_attr_store(struct kobject *kobj,
> struct attribute *attr,
> unsigned char *ptr = __struct_ptr(sbi, a->struct_type, a->offset);
> unsigned long t;
> int ret;
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + char *crypto_name;
> + size_t sz;
> +#endif
>
> switch (a->attr_id) {
> case attr_pointer_ui:
> @@ -181,6 +197,26 @@ static ssize_t erofs_attr_store(struct kobject *kobj,
> struct attribute *attr,
> if (t & 1)
> invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1);
> return len;
> +#endif
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + case attr_crypto:
> + buf = skip_spaces(buf);
> + sz = strlen(buf);
> + crypto_name = kstrdup(buf, GFP_KERNEL);
> + if (!crypto_name)
> + return -ENOMEM;
> +
> + /* ignore trailing newline */
> + if (sz > 0 && crypto_name[sz - 1] == '\n')
> + crypto_name[sz - 1] = 0x00;
> +
> + if (strlen(crypto_name) > 0) {
> + ret = z_erofs_crypto_enable_engine(crypto_name);
> + if (ret < 0)
> + return ret;
> + } else
> + z_erofs_crypto_disable_engine();
Is it possible to support disable a single engine?
Thanks,
Jianan
> + return len;
> #endif
> }
> return 0;
> --
> 2.31.1
>
>
