This is an automated email from the ASF dual-hosted git repository.
wombatu-kun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6f2fbaaf6e4b fix(storage-format): return all records when scanning
multi-block native HFiles (#19146)
6f2fbaaf6e4b is described below
commit 6f2fbaaf6e4bd58e8fb2c7eff9c604afc5ff25c3
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Jul 2 22:43:52 2026 -0700
fix(storage-format): return all records when scanning multi-block native
HFiles (#19146)
---
.../org/apache/hudi/io/hfile/HFileDataBlock.java | 6 +-
.../hudi/io/hfile/TestHFileMultiBlockScan.java | 94 ++++++++++++++++++++++
2 files changed, 98 insertions(+), 2 deletions(-)
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
index cbf3e719f6a0..551cfd2c961b 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
@@ -53,7 +53,9 @@ public class HFileDataBlock extends HFileBlock {
// so the latest timestamp is used.
private static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
- // End offset of content in the block, relative to the start of the start of
the block
+ // End offset of content in the block, relative to the start of the block.
The key-values
+ // occupy exactly uncompressedSizeWithoutHeader bytes after the header; the
checksum trails
+ // the content and is not part of it, so it must not be subtracted here.
protected final int uncompressedContentEndRelativeOffset;
private final List<KeyValueEntry> entriesToWrite = new ArrayList<>();
@@ -64,7 +66,7 @@ public class HFileDataBlock extends HFileBlock {
super(context, HFileBlockType.DATA, byteBuff, startOffsetInBuff);
this.uncompressedContentEndRelativeOffset =
- this.uncompressedEndOffset - this.sizeCheckSum -
this.startOffsetInBuff;
+ this.uncompressedEndOffset - this.startOffsetInBuff;
}
// For write purpose.
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
new file mode 100644
index 000000000000..e36410f856b5
--- /dev/null
+++
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.io.hfile;
+
+import org.apache.hudi.io.ByteArraySeekableDataInputStream;
+import org.apache.hudi.io.ByteBufferBackedInputStream;
+import org.apache.hudi.io.compress.CompressionCodec;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Locale;
+
+import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Reproduces a full-scan record drop on native-writer HFiles that span many
data blocks. A large
+ * block size packs many records per block; the trailer entry count is
correct, but a
+ * {@code seekTo()} then {@code next()} forward scan stops short of a block's
content end because
+ * the content-end bound subtracts the trailing checksum size. The drop only
surfaces once the
+ * checksum region exceeds the last record's size, which is why small-block
tests never caught it.
+ */
+public class TestHFileMultiBlockScan {
+
+ @ParameterizedTest
+ @CsvSource({
+ "NONE, 64, 5000", "GZIP, 64, 5000",
+ "NONE, 1048576, 200000", "GZIP, 1048576, 200000",
+ "NONE, 65536, 200000", "GZIP, 65536, 200000"
+ })
+ public void fullScanReturnsAllRecords(String codec, int blockSize, int
numRecords) throws Exception {
+ HFileContext context = new HFileContext.Builder()
+ .blockSize(blockSize)
+ .compressionCodec(CompressionCodec.valueOf(codec))
+ .build();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (HFileWriter writer = new HFileWriterImpl(context, baos)) {
+ for (int i = 0; i < numRecords; i++) {
+ writer.append(key(i), getUTF8Bytes(value(i)));
+ }
+ }
+ byte[] fileBytes = baos.toByteArray();
+
+ try (HFileReader reader = new HFileReaderImpl(
+ new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(fileBytes)),
+ fileBytes.length)) {
+ reader.initializeMetadata();
+ assertEquals(numRecords, reader.getNumKeyValueEntries(),
+ "trailer entry count for codec=" + codec + " blockSize=" +
blockSize);
+
+ String scenario = "codec=" + codec + " blockSize=" + blockSize + "
numRecords=" + numRecords;
+ int scanned = 0;
+ int firstSkipAt = -1;
+ boolean hasNext = reader.seekTo();
+ while (hasNext) {
+ KeyValue kv = reader.getKeyValue().get();
+ if (firstSkipAt < 0 &&
!key(scanned).equals(kv.getKey().getContentInString())) {
+ firstSkipAt = scanned;
+ }
+ scanned++;
+ hasNext = reader.next();
+ }
+ assertEquals(numRecords, scanned,
+ "forward scan returned fewer records than written; " + scenario
+ + " firstSkipAtPosition=" + firstSkipAt);
+ }
+ }
+
+ private static String key(int i) {
+ return String.format(Locale.ROOT, "%010d", i);
+ }
+
+ private static String value(int i) {
+ return String.format(Locale.ROOT, "value-%010d", i);
+ }
+}
