This is an automated email from the ASF dual-hosted git repository.
hubgeter pushed a commit to branch orc
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/orc by this push:
new 34a46c9e2de [improvement] Enable libdeflate for ZLIB decompression on
all architectures (#417)
34a46c9e2de is described below
commit 34a46c9e2de3214eea527f50c45c94264c14cdcd
Author: daidai <[email protected]>
AuthorDate: Tue Sep 29 10:34:48 2026 +0800
[improvement] Enable libdeflate for ZLIB decompression on all architectures
(#417)
ZlibDecompressionStreamByLibDeflate was only used on x86, so AArch64 fell
back
to the zlib inflate path for ORC ZLIB streams. libdeflate is built for every
platform in Doris' thirdparty, so drop the architecture guards and always
decode ZLIB chunks with libdeflate_deflate_decompress.
Add a test that malformed ZLIB chunks (output larger than the block buffer,
a
chunk longer than the stream, a truncated deflate payload and a reserved
deflate block type) raise ParseError.
---
c++/src/Compression.cc | 15 -------------
c++/test/TestCompression.cc | 53 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 53 insertions(+), 15 deletions(-)
diff --git a/c++/src/Compression.cc b/c++/src/Compression.cc
index 0354e99e01a..4b7f7f8f2e5 100644
--- a/c++/src/Compression.cc
+++ b/c++/src/Compression.cc
@@ -28,11 +28,7 @@
#include <iomanip>
#include <iostream>
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
#include <libdeflate.h>
-#endif
#include <sstream>
@@ -792,9 +788,6 @@ namespace orc {
outputBufferLength = 0;
}
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
class ZlibDecompressionStreamByLibDeflate : public BlockDecompressionStream {
public:
ZlibDecompressionStreamByLibDeflate(std::unique_ptr<SeekableInputStream>
inStream, size_t blockSize, MemoryPool& _pool,
@@ -827,7 +820,6 @@ namespace orc {
private:
libdeflate_decompressor* decompressor;
};
-#endif
class SnappyDecompressionStream : public BlockDecompressionStream {
public:
@@ -1230,15 +1222,8 @@ namespace orc {
case CompressionKind_NONE:
return input;
case CompressionKind_ZLIB:
-// Only used on x86 or x86_64
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) ||
defined(__i386__) || \
- defined(__i386) || defined(_M_IX86)
return
std::make_unique<ZlibDecompressionStreamByLibDeflate>(std::move(input),
blockSize, pool,
metrics);
-#else
- return std::make_unique<ZlibDecompressionStream>(std::move(input),
blockSize, pool,
- metrics);
-#endif
case CompressionKind_SNAPPY:
return std::make_unique<SnappyDecompressionStream>(std::move(input),
blockSize, pool,
metrics);
diff --git a/c++/test/TestCompression.cc b/c++/test/TestCompression.cc
index 2dba7b9a352..d8ddc24042a 100644
--- a/c++/test/TestCompression.cc
+++ b/c++/test/TestCompression.cc
@@ -369,4 +369,57 @@ namespace orc {
testSeekDecompressionStream(CompressionKind_LZ4);
testSeekDecompressionStream(CompressionKind_SNAPPY);
}
+
+ // ZLIB chunks are decoded by libdeflate, which must reject malformed chunks
instead of
+ // returning partial data.
+ TEST(Compression, zlib_malformed_chunks) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool* pool = getDefaultPool();
+ char testData[1000];
+ generateRandomData(testData, sizeof(testData), true);
+ compressAndVerify(CompressionKind_ZLIB, &memStream,
CompressionStrategy_SPEED, 1024, 1024,
+ *pool, testData, sizeof(testData));
+ const std::string compressed(memStream.getData(), memStream.getLength());
+ // A single chunk whose 3-byte header has the "original" bit cleared.
+ ASSERT_EQ(0, compressed[0] & 1);
+ ASSERT_EQ(compressed.size() - 3,
+ (static_cast<size_t>(static_cast<unsigned char>(compressed[0])) |
+ static_cast<size_t>(static_cast<unsigned char>(compressed[1]))
<< 8 |
+ static_cast<size_t>(static_cast<unsigned char>(compressed[2]))
<< 16) >>
+ 1);
+
+ auto decompressAll = [pool](const std::string& stream, uint64_t blockSize)
{
+ auto decompressStream = createDecompressor(
+ CompressionKind_ZLIB,
+ std::make_unique<SeekableArrayInputStream>(stream.data(),
stream.size()), blockSize,
+ *pool, getDefaultReaderMetrics());
+ std::string result;
+ const void* data;
+ int size;
+ while (decompressStream->Next(&data, &size)) {
+ result.append(static_cast<const char*>(data),
static_cast<size_t>(size));
+ }
+ return result;
+ };
+ EXPECT_EQ(std::string(testData, sizeof(testData)),
decompressAll(compressed, 1024));
+
+ // The decompressed chunk does not fit in the block buffer.
+ EXPECT_THROW(decompressAll(compressed, 512), ParseError);
+
+ // The chunk header claims more bytes than the stream holds.
+ EXPECT_THROW(decompressAll(compressed.substr(0, compressed.size() - 1),
1024), ParseError);
+
+ // The deflate payload is cut in half and the header is adjusted to match.
+ std::string truncated = compressed.substr(0, 3 + (compressed.size() - 3) /
2);
+ const size_t truncatedHeader = (truncated.size() - 3) << 1;
+ truncated[0] = static_cast<char>(truncatedHeader & 0xff);
+ truncated[1] = static_cast<char>((truncatedHeader >> 8) & 0xff);
+ truncated[2] = static_cast<char>((truncatedHeader >> 16) & 0xff);
+ EXPECT_THROW(decompressAll(truncated, 1024), ParseError);
+
+ // The first deflate block uses the reserved block type.
+ std::string corrupted = compressed;
+ corrupted[3] = 0x07;
+ EXPECT_THROW(decompressAll(corrupted, 1024), ParseError);
+ }
} // namespace orc
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
