This is an automated email from the ASF dual-hosted git repository.
yjhjstz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git
The following commit(s) were added to refs/heads/main by this push:
new 9135b8f8fd6 gpfdist add --compress-level
9135b8f8fd6 is described below
commit 9135b8f8fd65c2cb8405dcd6b79e1ec2b9b826a9
Author: Jianghua Yang <[email protected]>
AuthorDate: Wed Jun 18 23:36:48 2025 +0000
gpfdist add --compress-level
---
gpMgmt/doc/gpfdist_help | 11 +++++++-
src/bin/gpfdist/gpfdist.c | 29 +++++++++++++++++-----
.../gpfdist/regress/input/gpfdist2_compress.source | 2 +-
.../regress/output/gpfdist2_compress.source | 2 +-
4 files changed, 35 insertions(+), 9 deletions(-)
diff --git a/gpMgmt/doc/gpfdist_help b/gpMgmt/doc/gpfdist_help
index 4ad1f46ce83..cdb298662fc 100755
--- a/gpMgmt/doc/gpfdist_help
+++ b/gpMgmt/doc/gpfdist_help
@@ -10,7 +10,7 @@ SYNOPSIS
gpfdist [-d <directory>] [-p <http_port>] [-l <log_file>] [-t <timeout>]
[-S] [-w <time>] [-v | -V] [-m <max_length>] [--ssl <certificate_path>]
[--ssl_verify_peer <on/off>]
-[--compress]
+[--compress] [--compress-level]
gpfdist [-? | --help] | --version
@@ -162,6 +162,15 @@ OPTIONS
time-intensive process, which may potentially result in reduced
transmission speeds.
+--compress-level <level>
+
+ Specifies the compression level to be used with the Zstandard (zstd)
compression
+ when the --compress option is enabled. The <level> parameter is an integer
value
+ that controls the trade-off between compression ratio and speed. Higher values
+ typically result in better compression ratios but may require more CPU time.
+ The valid range is 1 (fastest, least compression) to 9 (slowest, best
compression).
+ The default compression level is 1 if not specified.
+
-v (verbose)
diff --git a/src/bin/gpfdist/gpfdist.c b/src/bin/gpfdist/gpfdist.c
index 350ef563abd..78fc2abc26b 100644
--- a/src/bin/gpfdist/gpfdist.c
+++ b/src/bin/gpfdist/gpfdist.c
@@ -198,7 +198,8 @@ static struct
const char* ssl; /* path to certificates in case we use gpfdist with
ssl */
int w; /* The time used for session timeout in
seconds */
int compress; /* The flag to indicate whether
comopression transmission is open */
-} opt = { 8080, 8080, 0, 0, 0, ".", 0, 0, -1, 5, 0, 32768, 0, 256, 0, 0, 0, 0,
0};
+ int compress_level; /* ZSTD compression level (1-9)
*/
+} opt = { 8080, 8080, 0, 0, 0, ".", 0, 0, -1, 5, 0, 32768, 0, 256, 0, 0, 0, 0,
0, DEFAULT_COMPRESS_LEVEL};
typedef union address
@@ -577,9 +578,9 @@ static void usage_error(const char* msg, int print_usage)
}
else
{
- fprintf(stderr,
+ fprintf(stderr,
"gpfdist -- file distribution web
server\n\n"
- "usage: gpfdist [--ssl
<certificates_directory>] [-d <directory>] [-p <http(s)_port>] [-l <log_file>]
[-t <timeout>] [-v | -V | -s] [-m <maxlen>] [-w <timeout>]"
+ "usage: gpfdist [--ssl
<certificates_directory>] [-d <directory>] [-p <http(s)_port>] [-l <log_file>]
[-t <timeout>] [-v | -V | -s] [-m <maxlen>] [-w <timeout>] [--compress]
[--compress-level <level>]"
#ifdef GPFXDIST
"[-c file]"
#endif
@@ -605,7 +606,12 @@ static void usage_error(const char* msg, int print_usage)
" -c file : configuration
file for transformations\n"
#endif
" --version : print
version information\n"
- " -w timeout : timeout
in seconds before close target file\n\n");
+ " -w timeout : timeout
in seconds before close target file\n"
+#ifdef USE_ZSTD
+ " --compress : enable
ZSTD compression\n"
+ " --compress-level :
ZSTD compression level (1-9, default=1)\n"
+#endif
+ "\n");
}
}
@@ -669,6 +675,7 @@ static void parse_command_line(int argc, const char* const
argv[],
{ "version", 256, 0, "print version number" },
{ NULL, 'w', 1, "wait for session timeout in seconds" },
{"compress", 258, 0, "turn on compressed transmission"},
+ {"compress-level", 259, 1, "ZSTD compression level (1-9, default=1)"},
{ 0 } };
status = apr_getopt_init(&os, pool, argc, argv);
@@ -757,10 +764,19 @@ static void parse_command_line(int argc, const char*
const argv[],
case 258:
opt.compress = 1;
break;
+ case 259:
+ opt.compress_level = atoi(arg);
+ if (opt.compress_level < 1 || opt.compress_level > 9) {
+ usage_error("Error: compression level must be
between 1 and 9", 0);
+ }
+ break;
#else
case 258:
usage_error("ZSTD is not supported by this build", 0);
break;
+ case 259:
+ usage_error("ZSTD compression level option is not
supported by this build", 0);
+ break;
#endif
}
}
@@ -4867,7 +4883,8 @@ static int compress_zstd(const request_t *r, block_t
*blk, int buflen)
return -1;
}
- size_t init_result = ZSTD_initCStream(r->zstd_cctx,
DEFAULT_COMPRESS_LEVEL);
+ /* Use configurable compression level */
+ size_t init_result = ZSTD_initCStream(r->zstd_cctx, opt.compress_level);
if (ZSTD_isError(init_result))
{
snprintf(r->zstd_error, r->zstd_err_len, "Creating compression
context initialization failed, error is %s.", ZSTD_getErrorName(init_result));
@@ -4905,7 +4922,7 @@ static int compress_zstd(const request_t *r, block_t
*blk, int buflen)
}
offset += output.pos;
- gdebug(NULL, "compress_zstd finished, input size = %d, output size =
%d.", buflen, offset);
+ gdebug(NULL, "compress_zstd finished, input size = %d, output size =
%d, compression level = %d.", buflen, offset, opt.compress_level);
return offset;
}
diff --git a/src/bin/gpfdist/regress/input/gpfdist2_compress.source
b/src/bin/gpfdist/regress/input/gpfdist2_compress.source
index ba9379fc9e5..7af836dddda 100644
--- a/src/bin/gpfdist/regress/input/gpfdist2_compress.source
+++ b/src/bin/gpfdist/regress/input/gpfdist2_compress.source
@@ -12,7 +12,7 @@ CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25),
R_COMMENT VARCHAR(152
DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start;
DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_stop;
CREATE EXTERNAL WEB TABLE gpfdist2_start (x text)
-execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data --compress
</dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070
>/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
+execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data --compress
--compress-level 3 </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl
127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
diff --git a/src/bin/gpfdist/regress/output/gpfdist2_compress.source
b/src/bin/gpfdist/regress/output/gpfdist2_compress.source
index a14f9c7a16d..27023a8acd2 100644
--- a/src/bin/gpfdist/regress/output/gpfdist2_compress.source
+++ b/src/bin/gpfdist/regress/output/gpfdist2_compress.source
@@ -12,7 +12,7 @@ CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25),
R_COMMENT VARCHAR(152
DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start;
DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_stop;
CREATE EXTERNAL WEB TABLE gpfdist2_start (x text)
-execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data --compress
</dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070
>/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
+execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data --compress
--compress-level 3 </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl
127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist2_stop (x text)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
