This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 855843f1b2 [GLUTEN-11283][VL] Respect spark default zstd compression
level (#11284)
855843f1b2 is described below
commit 855843f1b2519c382ae3f380f15e0f69025e307b
Author: Wechar Yu <[email protected]>
AuthorDate: Tue Dec 16 09:04:12 2025 +0800
[GLUTEN-11283][VL] Respect spark default zstd compression level (#11284)
---
cpp/velox/utils/VeloxWriterUtils.cc | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/cpp/velox/utils/VeloxWriterUtils.cc
b/cpp/velox/utils/VeloxWriterUtils.cc
index 9a9922423b..fc4b810fb8 100644
--- a/cpp/velox/utils/VeloxWriterUtils.cc
+++ b/cpp/velox/utils/VeloxWriterUtils.cc
@@ -35,7 +35,8 @@ using namespace facebook::velox::common;
namespace {
const int32_t kGzipWindowBits4k = 12;
-}
+const int32_t kZSTDDefaultCompressionLevel = 3;
+} // namespace
std::unique_ptr<WriterOptions> makeParquetWriteOption(const
std::unordered_map<std::string, std::string>& sparkConfs) {
int64_t maxRowGroupBytes = 134217728; // 128MB
@@ -73,12 +74,11 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const
std::unordered_map<s
compressionCodec = CompressionKind::CompressionKind_LZ4;
} else if (boost::iequals(compressionCodecStr, "zstd")) {
compressionCodec = CompressionKind::CompressionKind_ZSTD;
- if (auto it = sparkConfs.find(kParquetZSTDCompressionLevel); it !=
sparkConfs.end()) {
- auto compressionLevel = std::stoi(it->second);
- auto codecOptions =
std::make_shared<parquet::arrow::util::CodecOptions>();
- codecOptions->compression_level = compressionLevel;
- writeOption->codecOptions = std::move(codecOptions);
- }
+ auto codecOptions =
std::make_shared<parquet::arrow::util::CodecOptions>();
+ auto it = sparkConfs.find(kParquetZSTDCompressionLevel);
+ auto compressionLevel = it != sparkConfs.end() ? std::stoi(it->second) :
kZSTDDefaultCompressionLevel;
+ codecOptions->compression_level = compressionLevel;
+ writeOption->codecOptions = std::move(codecOptions);
} else if (boost::iequals(compressionCodecStr, "uncompressed")) {
compressionCodec = CompressionKind::CompressionKind_NONE;
} else if (boost::iequals(compressionCodecStr, "none")) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
