This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 33f0567dc03 [improvement](be) Enable libdeflate decompression for
Parquet GZIP and ORC ZLIB on all architectures (#68593)
33f0567dc03 is described below
commit 33f0567dc0323ba95ff11102dd3fd864b6d181cc
Author: daidai <[email protected]>
AuthorDate: Fri Oct 9 10:20:17 2026 +0800
[improvement](be) Enable libdeflate decompression for Parquet GZIP and ORC
ZLIB on all architectures (#68593)
### What problem does this PR solve?
Related PR: https://github.com/apache/doris-thirdparty/pull/417,
https://github.com/apache/doris-thirdparty/pull/418
Problem Summary:
Parquet GZIP pages and ORC ZLIB streams are decompressed with libdeflate
only on x86 (#27542, #27669). On AArch64 both fall back to zlib
`inflate`, which is several times slower. The x86-only restriction was
never needed:
- libdeflate is built for every platform by `build-thirdparty.sh`.
- The official `doris-thirdparty-prebuilt-linux-aarch64` package ships
an aarch64 `libdeflate.a`, reachable through the usual `installed/lib ->
lib64` link.
This PR:
- links `deflate` unconditionally in `be/cmake/thirdparty.cmake`;
- removes the architecture guards in
`be/src/util/block_compression.cpp`, so Parquet `GZIP` always uses
`GzipBlockCompressionByLibdeflate`;
- bumps `contrib/apache-orc` to include apache/doris-thirdparty#417,
which does the same for ORC `CompressionKind_ZLIB`
(`ZlibDecompressionStreamByLibDeflate`).
- makes `GzipBlockCompressionByLibdeflate` require a page to expand to
exactly the declared size.
- Before, an empty page returned OK even when the header declared
uncompressed bytes, and a page that expanded to fewer bytes than
declared was accepted. The Parquet readers only check the Status, so a
dictionary page with `compressed_page_size=0` could be decoded from an
unwritten buffer.
- Now empty input is accepted only for an empty output, and libdeflate
rejects short output (no `actual_out_nbytes_ret`).
- This also changes x86, where this codec was already used.
- bumps `contrib/apache-orc` to include apache/doris-thirdparty#418,
which allocates ORC's block decompression input buffer lazily, so
libdeflate ZLIB streams no longer hold an extra block per stream.
### Release note
Use libdeflate to decompress Parquet GZIP and ORC ZLIB data on ARM64.
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/cmake/thirdparty.cmake | 5 +--
be/src/util/block_compression.cpp | 34 ++++++------------
be/test/util/block_compression_test.cpp | 62 +++++++++++++++++++++++++++++++++
contrib/apache-orc | 2 +-
4 files changed, 74 insertions(+), 29 deletions(-)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 723d1f8bded..abc1c7c5f3d 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -190,10 +190,7 @@ if (OS_MACOSX)
add_thirdparty(intl)
endif()
-# Only used on x86 or x86_64
-if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR
"${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86_64")
- add_thirdparty(deflate)
-endif()
+add_thirdparty(deflate)
add_thirdparty(icuuc LIB64)
add_thirdparty(icui18n LIB64)
diff --git a/be/src/util/block_compression.cpp
b/be/src/util/block_compression.cpp
index 47c67a011a1..40ca6d17ad0 100644
--- a/be/src/util/block_compression.cpp
+++ b/be/src/util/block_compression.cpp
@@ -17,20 +17,13 @@
#include "util/block_compression.h"
+#include <brotli/decode.h>
#include <bzlib.h>
#include <gen_cpp/parquet_types.h>
#include <gen_cpp/segment_v2.pb.h>
-#include <glog/logging.h>
-
-#include <exception>
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
-#include <libdeflate.h>
-#endif
-#include <brotli/decode.h>
#include <glog/log_severity.h>
#include <glog/logging.h>
+#include <libdeflate.h>
#include <lz4/lz4.h>
#include <lz4/lz4frame.h>
#include <lz4/lz4hc.h>
@@ -43,6 +36,7 @@
#include <algorithm>
#include <cstdint>
+#include <exception>
#include <limits>
#include <mutex>
#include <orc/Exceptions.hh>
@@ -1443,9 +1437,6 @@ private:
const static int MEM_LEVEL = 8;
};
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
class GzipBlockCompressionByLibdeflate final : public GzipBlockCompression {
public:
GzipBlockCompressionByLibdeflate() : GzipBlockCompression() {}
@@ -1456,8 +1447,9 @@ public:
~GzipBlockCompressionByLibdeflate() override = default;
Status decompress(const Slice& input, Slice* output) override {
- if (input.empty()) {
- output->size = 0;
+ // Parquet page headers give the exact uncompressed size, so the page
must fill the
+ // output buffer. Without actual_out_nbytes_ret, libdeflate rejects
shorter output.
+ if (input.empty() && output->size == 0) {
return Status::OK();
}
thread_local std::unique_ptr<libdeflate_decompressor, void
(*)(libdeflate_decompressor*)>
@@ -1465,16 +1457,16 @@ public:
if (!decompressor) {
return Status::InternalError("libdeflate_alloc_decompressor
error.");
}
- std::size_t out_len;
auto result = libdeflate_gzip_decompress(decompressor.get(),
input.data, input.size,
- output->data, output->size,
&out_len);
+ output->data, output->size,
nullptr);
if (result != LIBDEFLATE_SUCCESS) {
- return Status::InternalError("libdeflate_gzip_decompress error,
res={}", result);
+ return Status::InternalError(
+ "libdeflate_gzip_decompress error, res={}, input size={},
output size={}",
+ result, input.size, output->size);
}
return Status::OK();
}
};
-#endif
class LzoBlockCompression final : public BlockCompressionCodec {
public:
@@ -1663,13 +1655,7 @@ Status
get_block_compression_codec(tparquet::CompressionCodec::type parquet_code
*codec = ZstdBlockCompression::instance();
break;
case tparquet::CompressionCodec::GZIP:
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
*codec = GzipBlockCompressionByLibdeflate::instance();
-#else
- *codec = GzipBlockCompression::instance();
-#endif
break;
case tparquet::CompressionCodec::LZO:
*codec = LzoBlockCompression::instance();
diff --git a/be/test/util/block_compression_test.cpp
b/be/test/util/block_compression_test.cpp
index 870345e6978..c7364a59ec9 100644
--- a/be/test/util/block_compression_test.cpp
+++ b/be/test/util/block_compression_test.cpp
@@ -149,6 +149,68 @@ TEST_F(BlockCompressionTest, multi) {
test_multi_slices(segment_v2::CompressionTypePB::ZSTD);
}
+// Parquet GZIP pages are decoded by libdeflate on every architecture, so
check it
+// against the zlib gzip codec and on the malformed inputs a data file can
contain.
+TEST_F(BlockCompressionTest, parquet_gzip) {
+ BlockCompressionCodec* codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::GZIP,
&codec).ok());
+ BlockCompressionCodec* zlib_codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(TFileCompressType::GZ,
&zlib_codec).ok());
+
+ std::string empty_output;
+ Slice empty_slice(empty_output);
+ EXPECT_TRUE(codec->decompress(Slice(), &empty_slice).ok());
+ EXPECT_EQ(0, empty_slice.size);
+
+ // An empty page must not satisfy a header that declares uncompressed
bytes.
+ std::string dict_output(4, '\0');
+ Slice dict_slice(dict_output);
+ EXPECT_FALSE(codec->decompress(Slice(), &dict_slice).ok());
+
+ for (size_t size : {1, 10, 1000, 65536, 1000000, 8 * 1024 * 1024}) {
+ SCOPED_TRACE(size);
+ // Repeat a short random string so that both literals and matches are
exercised.
+ std::string orig = generate_str(std::min<size_t>(size, 4096));
+ while (orig.size() < size) {
+ orig.append(orig, 0, std::min(orig.size(), size - orig.size()));
+ }
+ faststring compressed;
+ ASSERT_TRUE(zlib_codec->compress(orig, &compressed).ok());
+
+ std::string restored(size, '\0');
+ Slice output(restored);
+ ASSERT_TRUE(codec->decompress(Slice(compressed), &output).ok());
+ EXPECT_EQ(orig, restored);
+
+ Slice short_output(restored.data(), size - 1);
+ EXPECT_FALSE(codec->decompress(Slice(compressed), &short_output).ok());
+
+ // The page expands to fewer bytes than the header declares.
+ std::string larger(size + 1, '\0');
+ Slice long_output(larger);
+ EXPECT_FALSE(codec->decompress(Slice(compressed), &long_output).ok());
+
+ for (size_t len : {size_t {1}, size_t {10}, compressed.size() / 2,
compressed.size() - 4,
+ compressed.size() - 1}) {
+ output = Slice(restored);
+ EXPECT_FALSE(codec->decompress(Slice(compressed.data(), len),
&output).ok());
+ }
+
+ // Corrupt the gzip magic, a deflate byte and the CRC32 in the trailer.
+ for (size_t pos : {size_t {0}, compressed.size() / 2,
compressed.size() - 8}) {
+ faststring corrupted;
+ corrupted.assign_copy(compressed.data(), compressed.size());
+ corrupted.data()[pos] ^= 0x55;
+ output = Slice(restored);
+ EXPECT_FALSE(codec->decompress(Slice(corrupted), &output).ok());
+ }
+ }
+
+ std::string restored(16, '\0');
+ Slice output(restored);
+ EXPECT_FALSE(codec->decompress(Slice("not a gzip stream"), &output).ok());
+}
+
static void check_snappy_decompression(BlockCompressionCodec* codec, const
faststring& compressed,
const std::string& original) {
std::string restored(original.size(), '\0');
diff --git a/contrib/apache-orc b/contrib/apache-orc
index be0f1b73a7a..5860f94768b 160000
--- a/contrib/apache-orc
+++ b/contrib/apache-orc
@@ -1 +1 @@
-Subproject commit be0f1b73a7aeb78824a03e0dcb692c50a176d513
+Subproject commit 5860f94768b94406f31698ac66f4cc3d2a781324
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
