This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 0a7db9598de6 test(storage-format): cover point lookups and
incompressible values in the multi-block HFile scan test (#20547)
0a7db9598de6 is described below
commit 0a7db9598de6c7c21c1aa6ed387c3084070f20fe
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Nov 6 20:25:31 2026 -0800
test(storage-format): cover point lookups and incompressible values in the
multi-block HFile scan test (#20547)
With GZIP and compressible values the checksum stays shorter than the last
entry of a block, so the existing GZIP cases pass even when the content-end
bound drops trailing key-values. Add a GZIP case with incompressible values,
and look up every key with seekTo(key) on a fresh reader, so a regression in
either the scan or the point-lookup path fails regardless of codec.
Co-authored-by: Lin Liu <[email protected]>
---
.../hudi/io/hfile/TestHFileMultiBlockScan.java | 40 +++++++++++++++++++++-
1 file changed, 39 insertions(+), 1 deletion(-)
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
index e36410f856b5..fc1134293865 100644
---
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
+++
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileMultiBlockScan.java
@@ -27,9 +27,12 @@ import org.junit.jupiter.params.provider.CsvSource;
import java.io.ByteArrayOutputStream;
import java.util.Locale;
+import java.util.Random;
+import java.util.function.IntFunction;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Reproduces a full-scan record drop on native-writer HFiles that span many
data blocks. A large
@@ -47,6 +50,29 @@ public class TestHFileMultiBlockScan {
"NONE, 65536, 200000", "GZIP, 65536, 200000"
})
public void fullScanReturnsAllRecords(String codec, int blockSize, int
numRecords) throws Exception {
+ assertFullScanReturnsAllRecords(codec, blockSize, numRecords, i ->
getUTF8Bytes(value(i)));
+ }
+
+ /**
+ * GZIP masks the drop for compressible values: the checksum covers the
small on-disk block, so it
+ * stays shorter than the last record. With incompressible values the
on-disk block stays near the
+ * block size, the checksum grows past a short record, and every full block
loses its trailing entries.
+ */
+ @ParameterizedTest
+ @CsvSource({"GZIP, 1048576, 50000, 100"})
+ public void fullScanReturnsAllRecordsWithIncompressibleValues(
+ String codec, int blockSize, int numRecords, int valueLength) throws
Exception {
+ Random random = new Random(42);
+ byte[][] values = new byte[numRecords][];
+ for (int i = 0; i < numRecords; i++) {
+ values[i] = new byte[valueLength];
+ random.nextBytes(values[i]);
+ }
+ assertFullScanReturnsAllRecords(codec, blockSize, numRecords, i ->
values[i]);
+ }
+
+ private static void assertFullScanReturnsAllRecords(
+ String codec, int blockSize, int numRecords, IntFunction<byte[]>
valueOf) throws Exception {
HFileContext context = new HFileContext.Builder()
.blockSize(blockSize)
.compressionCodec(CompressionCodec.valueOf(codec))
@@ -54,7 +80,7 @@ public class TestHFileMultiBlockScan {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (HFileWriter writer = new HFileWriterImpl(context, baos)) {
for (int i = 0; i < numRecords; i++) {
- writer.append(key(i), getUTF8Bytes(value(i)));
+ writer.append(key(i), valueOf.apply(i));
}
}
byte[] fileBytes = baos.toByteArray();
@@ -82,6 +108,18 @@ public class TestHFileMultiBlockScan {
"forward scan returned fewer records than written; " + scenario
+ " firstSkipAtPosition=" + firstSkipAt);
}
+
+ // Seeks only move forward, so point lookups use a fresh reader.
+ try (HFileReader reader = new HFileReaderImpl(
+ new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(fileBytes)),
+ fileBytes.length)) {
+ reader.initializeMetadata();
+ assertTrue(reader.seekTo());
+ for (int i = 0; i < numRecords; i++) {
+ assertEquals(HFileReader.SEEK_TO_FOUND, reader.seekTo(new
UTF8StringKey(key(i))),
+ "point lookup missed key " + key(i) + "; codec=" + codec + "
blockSize=" + blockSize);
+ }
+ }
}
private static String key(int i) {
