This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2cfe6b96de79 fix(hfile): reject block sizes the HFile format cannot
represent (#20321)
2cfe6b96de79 is described below
commit 2cfe6b96de7958a5b631f58fea063d9c87f5db6b
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Oct 15 12:57:11 2026 -0700
fix(hfile): reject block sizes the HFile format cannot represent (#20321)
---
.../java/org/apache/hudi/io/hfile/HFileBlock.java | 46 +++++++-
.../java/org/apache/hudi/io/hfile/HFileCursor.java | 3 +-
.../org/apache/hudi/io/hfile/HFileDataBlock.java | 2 +
.../org/apache/hudi/io/hfile/HFileWriterImpl.java | 43 ++++++--
.../org/apache/hudi/io/hfile/TestHFileWriter.java | 34 ++++++
.../org/apache/hudi/io/hfile/TestHfileBlock.java | 121 +++++++++++++++++++++
6 files changed, 238 insertions(+), 11 deletions(-)
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
index 87fd8ed80f8c..e90e8692edc1 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
@@ -20,6 +20,7 @@
package org.apache.hudi.io.hfile;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.compress.CompressionCodec;
import lombok.AccessLevel;
@@ -63,6 +64,9 @@ public abstract class HFileBlock {
// KeyValue key suffix beyond the row: column-family length (1) + timestamp
(8) + type (1).
private static final int KEY_METADATA_SUFFIX_LENGTH = SIZEOF_BYTE +
SIZEOF_INT64 + SIZEOF_BYTE;
+ // A serialized block is held in one byte array, which the JVM caps just
below Integer.MAX_VALUE.
+ static final long MAX_SERIALIZED_BLOCK_SIZE = Integer.MAX_VALUE - 8;
+
static class Header {
// Format of header is:
// 8 bytes - block magic
@@ -122,7 +126,9 @@ public abstract class HFileBlock {
byteBuff, startOffsetInBuff +
Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX);
this.bytesPerChecksum = readInt(
byteBuff, startOffsetInBuff + Header.BYTES_PER_CHECKSUM_INDEX);
- this.sizeCheckSum = numChecksumBytes(getOnDiskSizeWithHeader(),
bytesPerChecksum);
+ this.sizeCheckSum = bytesPerChecksum > 0
+ ? numChecksumBytes((long) HFILEBLOCK_HEADER_SIZE +
onDiskSizeWithoutHeader, bytesPerChecksum) : 0;
+ validateHeader(byteBuff, startOffsetInBuff);
if (CompressionCodec.NONE.equals(context.getCompressionCodec())) {
isUnpacked = true;
this.startOffsetInBuff = startOffsetInBuff;
@@ -137,6 +143,28 @@ public abstract class HFileBlock {
this.startOffsetInBuff + HFILEBLOCK_HEADER_SIZE +
uncompressedSizeWithoutHeader;
}
+ /**
+ * Rejects impossible header sizes before they are used to allocate buffers
or locate content.
+ */
+ private void validateHeader(byte[] byteBuff, int startOffsetInBuff) {
+ long onDiskSizeWithHeader = (long) HFILEBLOCK_HEADER_SIZE +
onDiskSizeWithoutHeader;
+ long blockEndInBuff = startOffsetInBuff + onDiskSizeWithHeader;
+ long unpackedSize = (long) HFILEBLOCK_HEADER_SIZE +
uncompressedSizeWithoutHeader + sizeCheckSum;
+ boolean uncompressedExceedsOnDisk =
CompressionCodec.NONE.equals(context.getCompressionCodec())
+ && uncompressedSizeWithoutHeader > onDiskSizeWithoutHeader;
+ if (onDiskSizeWithoutHeader < 0 || uncompressedSizeWithoutHeader < 0 ||
bytesPerChecksum <= 0
+ || blockEndInBuff > byteBuff.length || uncompressedExceedsOnDisk
+ || onDiskDataSizeWithHeader < HFILEBLOCK_HEADER_SIZE ||
onDiskDataSizeWithHeader > onDiskSizeWithHeader
+ || unpackedSize > MAX_SERIALIZED_BLOCK_SIZE) {
+ throw new HoodieIOException(String.format(
+ "Corrupt %s block header at buffer offset %d:
onDiskSizeWithoutHeader=%d, "
+ + "uncompressedSizeWithoutHeader=%d,
onDiskDataSizeWithHeader=%d, bytesPerChecksum=%d, "
+ + "buffer length=%d",
+ blockType, startOffsetInBuff, onDiskSizeWithoutHeader,
uncompressedSizeWithoutHeader,
+ onDiskDataSizeWithHeader, bytesPerChecksum, byteBuff.length));
+ }
+ }
+
/**
* Initialize HFileBlock for write.
*/
@@ -168,6 +196,11 @@ public abstract class HFileBlock {
*/
public static HFileBlock parse(HFileContext context, byte[] byteBuff, int
startOffsetInBuff)
throws IOException {
+ if (startOffsetInBuff < 0 || byteBuff.length - startOffsetInBuff <
HFILEBLOCK_HEADER_SIZE) {
+ throw new HoodieIOException(String.format(
+ "Truncated HFile block header at buffer offset %d: %d bytes remain,
%d needed",
+ startOffsetInBuff, byteBuff.length - startOffsetInBuff,
HFILEBLOCK_HEADER_SIZE));
+ }
HFileBlockType blockType = HFileBlockType.parse(byteBuff,
startOffsetInBuff);
switch (blockType) {
case ROOT_INDEX:
@@ -295,8 +328,15 @@ public abstract class HFileBlock {
dataOutputStream.write(blockType.getMagic(), 0, 8);
// 2. onDiskSizeWithoutHeader.
int compressedDataSize = compressedBlockData.limit();
- int onDiskDataSizeWithHeader = HFileBlock.HFILEBLOCK_HEADER_SIZE +
compressedDataSize;
- int numChecksumBytes = numChecksumBytes(onDiskDataSizeWithHeader,
DEFAULT_BYTES_PER_CHECKSUM);
+ long onDiskDataSizeWithHeaderLong = (long)
HFileBlock.HFILEBLOCK_HEADER_SIZE + compressedDataSize;
+ int numChecksumBytes = numChecksumBytes(onDiskDataSizeWithHeaderLong,
DEFAULT_BYTES_PER_CHECKSUM);
+ if (onDiskDataSizeWithHeaderLong + numChecksumBytes >
MAX_SERIALIZED_BLOCK_SIZE) {
+ throw new HoodieIOException(String.format(
+ "%s block of %d compressed bytes with %d checksum bytes exceeds
the maximum serialized "
+ + "block size of %d bytes",
+ blockType, compressedDataSize, numChecksumBytes,
MAX_SERIALIZED_BLOCK_SIZE));
+ }
+ int onDiskDataSizeWithHeader = (int) onDiskDataSizeWithHeaderLong;
dataOutputStream.writeInt(compressedDataSize + numChecksumBytes);
// 3. uncompressedSizeWithoutHeader.
dataOutputStream.writeInt(uncompressedBlockData.limit());
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
index d629e7464ac7..c4c02ac72d74 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
@@ -34,7 +34,8 @@ public class HFileCursor {
private static final int INVALID_POSITION = -1;
- // Absolute position in the HFile, which can exceed Integer.MAX_VALUE for
files larger than 2GB
+ // Block start offset in the HFile plus the offset within the uncompressed
block. A long since
+ // the block start can exceed 2GB; the in-block part fits an int as a block
is under 2GB.
private long offset;
private Option<KeyValue> keyValue;
@Getter(AccessLevel.NONE)
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
index d655f6fafdd5..10855c917ccd 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
@@ -94,6 +94,7 @@ public class HFileDataBlock extends HFileBlock {
* lexicographically greater than the lookup key.
*/
int seekTo(HFileCursor cursor, Key key, long blockStartOffsetInFile) {
+ // Safe cast: a block is under 2GB, so an offset within it fits an int.
int relativeOffset = (int) (cursor.getOffset() - blockStartOffsetInFile);
int lastRelativeOffset = relativeOffset;
Option<KeyValue> lastKeyValue = cursor.getKeyValue();
@@ -159,6 +160,7 @@ public class HFileDataBlock extends HFileBlock {
* @return {@code true} if there is next {@link KeyValue}; {code false}
otherwise.
*/
boolean next(HFileCursor cursor, long blockStartOffsetInFile) {
+ // Safe cast: a block is under 2GB, so an offset within it fits an int.
int offset = (int) (cursor.getOffset() - blockStartOffsetInFile);
Option<KeyValue> keyValue = cursor.getKeyValue();
if (!keyValue.isPresent()) {
diff --git
a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileWriterImpl.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileWriterImpl.java
index 348d1b06b661..934f25fb9403 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileWriterImpl.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileWriterImpl.java
@@ -50,6 +50,9 @@ public class HFileWriterImpl implements HFileWriter {
// + 2 bytes (length of key length)
// + 10 bytes (check sum, timestamp, key type).
private static final int EXTRA_BYTES_PER_DATA_ENTRY = 21;
+ // Block sizes are 4-byte ints. The 2 MiB headroom covers the header,
checksums (< 512 KiB) and
+ // deflate growth of incompressible data (< 660 KiB); HFileBlock#serialize
checks the exact bound.
+ static final long MAX_UNCOMPRESSED_DATA_BLOCK_BYTES = Integer.MAX_VALUE -
(2L << 20);
private final OutputStream outputStream;
private final HFileContext context;
// Meta Info map.
@@ -65,6 +68,7 @@ public class HFileWriterImpl implements HFileWriter {
private long currentOffset;
private long loadOnOpenSectionOffset;
private final int blockSize;
+ private final long maxUncompressedDataBlockBytes;
// Variables used to record necessary information to reduce
// the memory usage.
@@ -76,9 +80,17 @@ public class HFileWriterImpl implements HFileWriter {
private long totalValueLength = 0;
public HFileWriterImpl(HFileContext context, OutputStream outputStream) {
+ this(context, outputStream, MAX_UNCOMPRESSED_DATA_BLOCK_BYTES);
+ }
+
+ /**
+ * @param maxUncompressedDataBlockBytes max uncompressed bytes per data
block, lowered in tests.
+ */
+ HFileWriterImpl(HFileContext context, OutputStream outputStream, long
maxUncompressedDataBlockBytes) {
this.outputStream = outputStream;
this.context = context;
- this.blockSize = this.context.getBlockSize();
+ this.maxUncompressedDataBlockBytes = maxUncompressedDataBlockBytes;
+ this.blockSize = (int) Math.min(this.context.getBlockSize(),
maxUncompressedDataBlockBytes);
this.uncompressedDataBlockBytes = 0L;
this.totalUncompressedDataBlockBytes = 0L;
this.currentOffset = 0L;
@@ -92,19 +104,36 @@ public class HFileWriterImpl implements HFileWriter {
// Append a data kv pair.
public void append(String key, byte[] value) throws IOException {
byte[] keyBytes = StringUtils.getUTF8Bytes(key);
+ long entryBytes = (long) keyBytes.length + value.length +
EXTRA_BYTES_PER_DATA_ENTRY;
+ // Records with the same key must be put into the same block.
+ boolean flushBeforeAppend =
!Arrays.equals(currentDataBlock.getLastKeyContent(), keyBytes)
+ && uncompressedDataBlockBytes + entryBytes > blockSize;
+ // Check before mutating state so a rejected record leaves no trace.
+ checkDataBlockCapacity((flushBeforeAppend ? 0 :
uncompressedDataBlockBytes) + entryBytes, key);
lastKey = keyBytes;
totalKeyLength += keyBytes.length;
totalValueLength += value.length;
- // Records with the same key must be put into the same block.
- if (!Arrays.equals(currentDataBlock.getLastKeyContent(), keyBytes)
- && uncompressedDataBlockBytes + keyBytes.length + value.length +
EXTRA_BYTES_PER_DATA_ENTRY > blockSize) {
+ if (flushBeforeAppend) {
flushCurrentDataBlock();
uncompressedDataBlockBytes = 0;
}
currentDataBlock.add(keyBytes, value);
- int uncompressedKeyValueSize = keyBytes.length + value.length;
- uncompressedDataBlockBytes += uncompressedKeyValueSize +
EXTRA_BYTES_PER_DATA_ENTRY;
- totalUncompressedDataBlockBytes += uncompressedKeyValueSize +
EXTRA_BYTES_PER_DATA_ENTRY;
+ uncompressedDataBlockBytes += entryBytes;
+ totalUncompressedDataBlockBytes += entryBytes;
+ }
+
+ /**
+ * Rejects an append that would grow the block past its capacity, which only
an oversized record
+ * or a run of same-key records can do.
+ */
+ private void checkDataBlockCapacity(long uncompressedBytes, String key) {
+ if (uncompressedBytes > maxUncompressedDataBlockBytes) {
+ throw new IllegalArgumentException(String.format(
+ "Data block would hold %d uncompressed bytes after appending key %s,
exceeding the "
+ + "maximum of %d bytes: a single record, or records sharing the
same key, cannot "
+ + "exceed the maximum size of one data block",
+ uncompressedBytes, key, maxUncompressedDataBlockBytes));
+ }
}
// Append a metadata kv pair.
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
index e33cf65389fe..a206a76943b4 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
@@ -66,6 +66,7 @@ import static
org.apache.hudi.io.hfile.HFileInfo.MAX_MVCC_TS_KEY;
import static org.apache.hudi.io.hfile.HFileTrailer.TRAILER_SIZE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class TestHFileWriter {
@@ -98,6 +99,39 @@ class TestHFileWriter {
+
"616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161"
+
"616161616161616161616161616161616161616161007fffffffffffffff0400000000";
+ /**
+ * Same-key and oversized records beyond the block capacity are rejected
without changing the
+ * writer's state.
+ */
+ @Test
+ void testAppendRejectsRecordsExceedingDataBlockCapacity() throws IOException
{
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ // One entry costs 21 bytes of framing plus key and value
+ byte[] fittingValue = new byte[50];
+ byte[] oversizedValue = new byte[100];
+ try (HFileWriter writer = new HFileWriterImpl(CONTEXT, out, 100L)) {
+ writer.append("a", fittingValue);
+ IllegalArgumentException sameKey =
assertThrows(IllegalArgumentException.class,
+ () -> writer.append("a", fittingValue));
+ assertTrue(sameKey.getMessage().startsWith(
+ "Data block would hold 144 uncompressed bytes after appending key a,
exceeding the maximum of 100 bytes"),
+ sameKey.getMessage());
+ IllegalArgumentException oversized =
assertThrows(IllegalArgumentException.class,
+ () -> writer.append("bbbbb", oversizedValue));
+ assertTrue(oversized.getMessage().startsWith(
+ "Data block would hold 126 uncompressed bytes after appending key
bbbbb, exceeding the maximum of 100 bytes"),
+ oversized.getMessage());
+ }
+ byte[] hfile = out.toByteArray();
+ try (HFileReader reader = new HFileReaderImpl(
+ new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(hfile)), hfile.length)) {
+ assertEquals(1, reader.getNumKeyValueEntries());
+ assertEquals("a", new
Key(reader.getMetaInfo(LAST_KEY).get()).getContentInString());
+ assertEquals(1, IOUtils.readInt(reader.getMetaInfo(AVG_KEY_LEN).get(),
0));
+ assertEquals(fittingValue.length,
IOUtils.readInt(reader.getMetaInfo(AVG_VALUE_LEN).get(), 0));
+ }
+ }
+
@Test
void testOverflow() throws Exception {
// 1. Write data.
diff --git a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHfileBlock.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHfileBlock.java
index 9bb57d11cacc..15473ff6927f 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHfileBlock.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHfileBlock.java
@@ -19,12 +19,133 @@
package org.apache.hudi.io.hfile;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.io.compress.CompressionCodec;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import static org.apache.hudi.io.hfile.HFileBlock.HFILEBLOCK_HEADER_SIZE;
+import static
org.apache.hudi.io.hfile.HFileBlock.Header.BYTES_PER_CHECKSUM_INDEX;
+import static
org.apache.hudi.io.hfile.HFileBlock.Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX;
+import static
org.apache.hudi.io.hfile.HFileBlock.Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX;
+import static
org.apache.hudi.io.hfile.HFileBlock.Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
class TestHfileBlock {
+ private static final HFileContext CONTEXT = HFileContext.builder().build();
+
+ @Test
+ void testParseDataBlock() throws IOException {
+ HFileBlock block = HFileBlock.parse(CONTEXT, serializedDataBlock(), 0);
+ assertEquals(HFileBlockType.DATA, block.getBlockType());
+ }
+
+ @Test
+ void testParseRejectsTruncatedHeader() throws IOException {
+ byte[] bytes = serializedDataBlock();
+ // A header-only buffer describing an empty block is the smallest
parseable input
+ byte[] headerOnly = Arrays.copyOf(bytes, HFILEBLOCK_HEADER_SIZE);
+ ByteBuffer.wrap(headerOnly).putInt(ON_DISK_SIZE_WITHOUT_HEADER_INDEX, 0)
+ .putInt(UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX, 0)
+ .putInt(ON_DISK_DATA_SIZE_WITH_HEADER_INDEX, HFILEBLOCK_HEADER_SIZE);
+ assertEquals(HFileBlockType.DATA, HFileBlock.parse(CONTEXT, headerOnly,
0).getBlockType());
+ assertTruncatedHeader(Arrays.copyOf(bytes, HFILEBLOCK_HEADER_SIZE - 1), 0);
+ assertTruncatedHeader(bytes, bytes.length - HFILEBLOCK_HEADER_SIZE + 1);
+ }
+
+ @Test
+ void testParseRejectsNegativeSize() throws IOException {
+ for (int headerIndex : new int[] {ON_DISK_SIZE_WITHOUT_HEADER_INDEX,
UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX}) {
+ byte[] bytes = serializedDataBlock();
+ ByteBuffer.wrap(bytes).putInt(headerIndex, -1);
+ assertCorruptHeader(bytes);
+ }
+ }
+
+ @Test
+ void testParseRejectsOnDiskSizeBeyondBuffer() throws IOException {
+ byte[] bytes = serializedDataBlock();
+ ByteBuffer.wrap(bytes).putInt(ON_DISK_SIZE_WITHOUT_HEADER_INDEX,
bytes.length);
+ assertCorruptHeader(bytes);
+ }
+
+ @Test
+ void testParseRejectsZeroBytesPerChecksum() throws IOException {
+ byte[] bytes = serializedDataBlock();
+ ByteBuffer.wrap(bytes).putInt(BYTES_PER_CHECKSUM_INDEX, 0);
+ assertCorruptHeader(bytes);
+ }
+
+ @Test
+ void testParseRejectsUncompressedSizeBeyondOnDiskSizeWithoutCompression()
throws IOException {
+ byte[] bytes = serializedDataBlock();
+ int onDiskSize =
ByteBuffer.wrap(bytes).getInt(ON_DISK_SIZE_WITHOUT_HEADER_INDEX);
+ ByteBuffer.wrap(bytes).putInt(UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX,
onDiskSize + 1);
+ assertCorruptHeader(bytes);
+ }
+
+ @Test
+ void testParseRejectsOnDiskDataSizeOutsideBlock() throws IOException {
+ HFileContext gzipContext =
HFileContext.builder().compressionCodec(CompressionCodec.GZIP).build();
+ byte[] bytes = serializedDataBlock(gzipContext);
+ int onDiskSizeWithHeader =
+ HFILEBLOCK_HEADER_SIZE +
ByteBuffer.wrap(bytes).getInt(ON_DISK_SIZE_WITHOUT_HEADER_INDEX);
+ for (int atBound : new int[] {HFILEBLOCK_HEADER_SIZE,
onDiskSizeWithHeader}) {
+ byte[] copy = bytes.clone();
+ ByteBuffer.wrap(copy).putInt(ON_DISK_DATA_SIZE_WITH_HEADER_INDEX,
atBound);
+ assertEquals(HFileBlockType.DATA, HFileBlock.parse(gzipContext, copy,
0).getBlockType());
+ }
+ for (int outOfBound : new int[] {HFILEBLOCK_HEADER_SIZE - 1,
onDiskSizeWithHeader + 1}) {
+ byte[] copy = bytes.clone();
+ ByteBuffer.wrap(copy).putInt(ON_DISK_DATA_SIZE_WITH_HEADER_INDEX,
outOfBound);
+ assertCorruptHeader(gzipContext, copy);
+ }
+ }
+
+ @Test
+ void testParseRejectsUncompressedSizeTooLargeForOneArray() throws
IOException {
+ HFileContext gzipContext =
HFileContext.builder().compressionCodec(CompressionCodec.GZIP).build();
+ byte[] bytes = serializedDataBlock(gzipContext);
+ ByteBuffer.wrap(bytes).putInt(UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX,
Integer.MAX_VALUE - 10);
+ assertCorruptHeader(gzipContext, bytes);
+ }
+
+ private static byte[] serializedDataBlock() throws IOException {
+ return serializedDataBlock(CONTEXT);
+ }
+
+ private static byte[] serializedDataBlock(HFileContext context) throws
IOException {
+ HFileDataBlock block = HFileDataBlock.createDataBlockToWrite(context, -1L);
+ block.add("key".getBytes(StandardCharsets.UTF_8),
"value".getBytes(StandardCharsets.UTF_8));
+ ByteBuffer buffer = block.serialize();
+ return Arrays.copyOf(buffer.array(), buffer.limit());
+ }
+
+ private static void assertCorruptHeader(byte[] bytes) {
+ assertCorruptHeader(CONTEXT, bytes);
+ }
+
+ private static void assertCorruptHeader(HFileContext context, byte[] bytes) {
+ HoodieIOException e = assertThrows(HoodieIOException.class,
+ () -> HFileBlock.parse(context, bytes, 0));
+ assertTrue(e.getMessage().startsWith("Corrupt DATA block header at buffer
offset 0:"), e.getMessage());
+ }
+
+ private static void assertTruncatedHeader(byte[] bytes, int startOffset) {
+ HoodieIOException e = assertThrows(HoodieIOException.class,
+ () -> HFileBlock.parse(CONTEXT, bytes, startOffset));
+ assertEquals(String.format("Truncated HFile block header at buffer offset
%d: %d bytes remain, %d needed",
+ startOffset, bytes.length - startOffset, HFILEBLOCK_HEADER_SIZE),
e.getMessage());
+ }
+
@Test
void testNumChecksumChunksZeroBytes() {
Assertions.assertEquals(0, HFileBlock.numChecksumChunks(0L, 512));
