This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cfd14629af2 Fix re-reading a huge value with the same
VarByteChunkForwardIndexReaderV4 context (#19638)
cfd14629af2 is described below
commit cfd14629af2315f8aa9e035f5d180f39c1e53453
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed Sep 23 09:59:42 2026 -0700
Fix re-reading a huge value with the same VarByteChunkForwardIndexReaderV4
context (#19638)
---
.../forward/VarByteChunkForwardIndexReaderV4.java | 3 +-
.../segment/index/creator/VarByteChunkV4Test.java | 66 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
index a7b6bb5b4d7..b3a74be0378 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
@@ -290,7 +290,8 @@ public class VarByteChunkForwardIndexReaderV4
}
public byte[] getValue(int docId) {
- if (docId >= _docIdOffset && docId < _nextDocIdOffset) {
+ // A huge chunk holds a single value without the regular chunk header,
so it is read again on every access
+ if (_regularChunk && docId >= _docIdOffset && docId < _nextDocIdOffset) {
return readSmallUncompressedValue(docId);
} else {
try {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/VarByteChunkV4Test.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/VarByteChunkV4Test.java
index 6b6ed9336d2..c3f8cdeb2a5 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/VarByteChunkV4Test.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/VarByteChunkV4Test.java
@@ -79,6 +79,17 @@ public class VarByteChunkV4Test implements
PinotBuffersAfterClassCheckRule {
return params;
}
+ @DataProvider
+ public Object[][] compressionTypes() {
+ return new Object[][]{
+ {ChunkCompressionType.PASS_THROUGH},
+ {ChunkCompressionType.LZ4},
+ {ChunkCompressionType.LZ4_LENGTH_PREFIXED},
+ {ChunkCompressionType.SNAPPY},
+ {ChunkCompressionType.ZSTANDARD}
+ };
+ }
+
protected String getTestDirName() {
return "VarByteChunkV4Test";
}
@@ -193,6 +204,61 @@ public class VarByteChunkV4Test implements
PinotBuffersAfterClassCheckRule {
FileUtils.deleteQuietly(mapSVFile);
}
+ /// A value larger than the chunk size is stored alone in a huge chunk,
which has no regular chunk header. Reading
+ /// the same doc repeatedly with one context returns the written value every
time.
+ @Test(dataProvider = "compressionTypes")
+ public void testHugeValueReadTwiceSV(ChunkCompressionType compressionType)
+ throws IOException {
+ File file = new File(_dirs[0], "testHugeValueReadTwiceSV" +
compressionType);
+ int chunkSize = 1024;
+ String[] values = {"small", "huge".repeat(chunkSize), "small"};
+ try (VarByteChunkWriter writer = createWriter(file, compressionType,
chunkSize)) {
+ for (String value : values) {
+ writer.putString(value);
+ }
+ }
+ try (PinotDataBuffer buffer =
PinotDataBuffer.mapReadOnlyBigEndianFile(file);
+ VarByteChunkForwardIndexReaderV4 reader = createReader(buffer,
FieldSpec.DataType.STRING, true);
+ VarByteChunkForwardIndexReaderV4.ReaderContext context =
reader.createContext()) {
+ for (int docId = 0; docId < values.length; docId++) {
+ assertEquals(reader.getString(docId, context), values[docId]);
+ assertEquals(reader.getString(docId, context), values[docId]);
+ }
+ }
+ FileUtils.deleteQuietly(file);
+ }
+
+ /// Multi-value counterpart of [#testHugeValueReadTwiceSV], reading the huge
doc through each MV accessor in turn.
+ @Test(dataProvider = "compressionTypes")
+ public void testHugeValueReadTwiceMV(ChunkCompressionType compressionType)
+ throws IOException {
+ File file = new File(_dirs[0], "testHugeValueReadTwiceMV" +
compressionType);
+ int chunkSize = 1024;
+ String[] hugeValue = new String[chunkSize];
+ for (int i = 0; i < hugeValue.length; i++) {
+ hugeValue[i] = "huge-" + i;
+ }
+ String[][] values = {{"small"}, hugeValue, {"small", "values"}};
+ try (VarByteChunkWriter writer = createWriter(file, compressionType,
chunkSize)) {
+ for (String[] value : values) {
+ writer.putStringMV(value);
+ }
+ }
+ try (PinotDataBuffer buffer =
PinotDataBuffer.mapReadOnlyBigEndianFile(file);
+ VarByteChunkForwardIndexReaderV4 reader = createReader(buffer,
FieldSpec.DataType.STRING, false);
+ VarByteChunkForwardIndexReaderV4.ReaderContext context =
reader.createContext()) {
+ String[] valueBuffer = new String[hugeValue.length];
+ for (int docId = 0; docId < values.length; docId++) {
+ String[] expected = values[docId];
+ assertEquals(reader.getNumValuesMV(docId, context), expected.length);
+ assertEquals(reader.getStringMV(docId, context), expected);
+ assertEquals(reader.getStringMV(docId, valueBuffer, context),
expected.length);
+ assertEquals(Arrays.copyOf(valueBuffer, expected.length), expected);
+ }
+ }
+ FileUtils.deleteQuietly(file);
+ }
+
static class StringSplitterMV implements Function<String, String[]> {
@Override
public String[] apply(String input) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
