This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8e0402db7cc7 fix(hfile): bound compressed block decompression (#19930)
8e0402db7cc7 is described below
commit 8e0402db7cc7264b5426ed01c3de7c0ba4ae4956
Author: Sherwin Tan <[email protected]>
AuthorDate: Mon Oct 5 18:48:51 2026 -0700
fix(hfile): bound compressed block decompression (#19930)
* fix(hfile): bound compressed block decompression
Exclude trailing checksum bytes from the native GZIP input stream and add a
regression test for Hudi issue #19929.
* test(hfile): cover HBase checksum tails in decompression regression
* test(hfile): reuse reader fixture for decompression regression
---------
Co-authored-by: danny0405 <[email protected]>
---
.../java/org/apache/hudi/io/hfile/HFileBlock.java | 17 +++++++++++++++--
.../org/apache/hudi/io/hfile/TestHFileReader.java | 6 ++++++
.../hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile | Bin 101870 -> 101870 bytes
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
index ae99ef2f1cf7..87fd8ed80f8c 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
@@ -92,6 +92,7 @@ public abstract class HFileBlock {
@Getter
private final HFileBlockType blockType;
protected final int onDiskSizeWithoutHeader;
+ protected final int onDiskDataSizeWithHeader;
protected final int uncompressedSizeWithoutHeader;
protected final int bytesPerChecksum;
private boolean isUnpacked = false;
@@ -115,6 +116,8 @@ public abstract class HFileBlock {
this.blockType = blockType;
this.onDiskSizeWithoutHeader = readInt(
byteBuff, startOffsetInBuff +
Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX);
+ this.onDiskDataSizeWithHeader = readInt(
+ byteBuff, startOffsetInBuff +
Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
this.uncompressedSizeWithoutHeader = readInt(
byteBuff, startOffsetInBuff +
Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX);
this.bytesPerChecksum = readInt(
@@ -149,6 +152,7 @@ public abstract class HFileBlock {
this.sizeCheckSum = -1;
this.uncompressedEndOffset = -1;
this.onDiskSizeWithoutHeader = -1;
+ this.onDiskDataSizeWithHeader = -1;
this.uncompressedSizeWithoutHeader = -1;
this.bytesPerChecksum = -1;
}
@@ -232,13 +236,22 @@ public abstract class HFileBlock {
// Copy the block header which is not compressed
System.arraycopy(
compressedByteBuff, startOffsetInCompressedBuff, byteBuff, 0,
HFILEBLOCK_HEADER_SIZE);
+ // onDiskSizeWithoutHeader includes the trailing checksum bytes, while
+ // onDiskDataSizeWithHeader ends immediately after the compressed
payload.
+ // Passing the former to GZIPInputStream lets it interpret checksum
bytes as
+ // another gzip member and can result in ZipException (for example,
during
+ // metadata table compaction).
+ // byteBuff also reserves sizeCheckSum bytes, so limit output to
uncompressedSizeWithoutHeader.
+ // Using byteBuff.length - HFILEBLOCK_HEADER_SIZE makes readFully keep
reading beyond
+ // the first gzip stream to fill that checksum space.
+ int compressedDataSize = onDiskDataSizeWithHeader -
HFILEBLOCK_HEADER_SIZE;
try (InputStream byteBuffInputStream = new ByteArrayInputStream(
- compressedByteBuff, startOffsetInCompressedBuff +
HFILEBLOCK_HEADER_SIZE, onDiskSizeWithoutHeader)) {
+ compressedByteBuff, startOffsetInCompressedBuff +
HFILEBLOCK_HEADER_SIZE, compressedDataSize)) {
context.getCompressor().decompress(
byteBuffInputStream,
byteBuff,
HFILEBLOCK_HEADER_SIZE,
- byteBuff.length - HFILEBLOCK_HEADER_SIZE);
+ uncompressedSizeWithoutHeader);
}
}
isUnpacked = true;
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
index 31259dce1f71..255502c569f4 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
@@ -251,6 +251,12 @@ public class TestHFileReader {
new KeyLookUpInfo("hudi-key-000019999b", SEEK_TO_EOF, "", "")
)
),
+ // The first two data blocks have 12 checksum bytes each, replaced
with gzip-like tails:
+ // block 0: 1f8b08000000000000000700 (invalid DEFLATE block type),
+ // block 1: 1f8b08000000000000000000 (truncated DEFLATE block).
+ // The native reader does not validate CRCs, but must exclude these
bytes from decompression.
+ // All headers and compressed payloads are unchanged. This exercises
both ZipException and
+ // EOFException regressions through the existing sequential, point,
and prefix lookup checks.
Arguments.of(
"/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile",
20000,
diff --git
a/hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile
b/hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile
index 4d35a3ec8868..a40abe4e266b 100644
Binary files
a/hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile
and
b/hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile
differ
