This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch cs_UwWULRT7lt/codec-stack-06-reload-enable
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 35ce901c451e464338d3a7988a7b25634aaafc08
Author: intentlab-ai[bot] <intentlab-ai[bot]@users.noreply.github.com>
AuthorDate: Sun Sep 20 09:14:59 2026 +0000
Report uncompressed value size for V7 codec-pipeline columns
This PR enabled compression statistics for V7 columns (the metadata and
reader no longer require a legacy ChunkCompressionType), but the creator
still returned -1 for the V7 writer's uncompressed value size. That -1 maps
to UNAVAILABLE, so codecSpec columns never reported compression stats at
creation or on reload-to-V7, and testV7SegmentCreationPersistsAndReports-
CompressionStats failed.
The V7 fixed-byte layout stores exactly one value per doc, so the
uncompressed size is totalDocs * valueType.size(). Report it when tracking
is enabled, mirroring the legacy writer's contract. Update the V7 round-trip
assertion in ForwardIndexCreatorFactoryTest, which encoded the old -1.
---
.../impl/fwd/SingleValueFixedByteRawIndexCreator.java | 16 +++++++++++++++-
.../index/forward/ForwardIndexCreatorFactoryTest.java | 6 ++++--
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
index e7758280116..0a6d7795109 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
@@ -43,6 +43,12 @@ public class SingleValueFixedByteRawIndexCreator implements
CompressionStatsTrac
private final DataType _valueType;
@Nullable
private final ChunkCompressionType _chunkCompressionType;
+ // Number of documents the writer will be fed. Used to compute the
uncompressed value size for the
+ // V7 codec-pipeline writer, whose fixed-byte layout stores exactly
totalDocs * valueType.size() bytes.
+ private final int _totalDocs;
+ // Whether compression-statistics tracking was requested. Mirrors the legacy
writer's contract of
+ // reporting an uncompressed value size only when tracking is enabled.
+ private boolean _trackUncompressedValueSize;
/// Constructor for the class
///
@@ -77,6 +83,7 @@ public class SingleValueFixedByteRawIndexCreator implements
CompressionStatsTrac
writerVersion);
_valueType = valueType;
_chunkCompressionType = compressionType;
+ _totalDocs = totalDocs;
}
/// Creates a raw fixed-byte creator backed by the V7 codec-pipeline writer.
@@ -95,6 +102,7 @@ public class SingleValueFixedByteRawIndexCreator implements
CompressionStatsTrac
valueType.size());
_valueType = valueType;
_chunkCompressionType = null;
+ _totalDocs = totalDocs;
}
@Override
@@ -140,10 +148,15 @@ public class SingleValueFixedByteRawIndexCreator
implements CompressionStatsTrac
@Override
public long getRawForwardIndexUncompressedValueSizeInBytes() {
- // Compression-statistics metadata supports only the legacy
single-compressor format.
if (_indexWriter instanceof FixedByteChunkForwardIndexWriter legacyWriter)
{
return legacyWriter.getRawForwardIndexUncompressedValueSizeInBytes();
}
+ // V7 codec-pipeline writer: the fixed-byte layout stores exactly one
value per doc, so the
+ // uncompressed size is totalDocs * valueType.size(). Report it only when
tracking was requested,
+ // matching the legacy writer's contract of returning -1 otherwise.
+ if (_trackUncompressedValueSize) {
+ return (long) _totalDocs * _valueType.size();
+ }
return -1;
}
@@ -155,6 +168,7 @@ public class SingleValueFixedByteRawIndexCreator implements
CompressionStatsTrac
@Override
public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _trackUncompressedValueSize = true;
if (_indexWriter instanceof FixedByteChunkForwardIndexWriter legacyWriter)
{
legacyWriter.enableRawForwardIndexUncompressedValueSizeTracking();
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
index 27143342fab..1e1125c89ad 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
@@ -140,7 +140,8 @@ public class ForwardIndexCreatorFactoryTest {
.withTargetDocsPerChunk(2)
.build();
TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
- // Legacy compression stats are opted in here on purpose: a codecSpec
column never reports them.
+ // Compression stats are opted in here on purpose: a V7 codecSpec column
reports its uncompressed
+ // value size (totalDocs * storedType.size()) even though it has no
legacy ChunkCompressionType.
tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
long[] values = storedType == DataType.INT
? new long[]{11, 13, 21}
@@ -157,7 +158,8 @@ public class ForwardIndexCreatorFactoryTest {
}
}
creator.seal();
- assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(),
-1L);
+ assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ (long) values.length * storedType.size());
}
File indexFile = new File(indexDir, COLUMN_NAME +
V1Constants.Indexes.RAW_SV_FORWARD_INDEX_FILE_EXTENSION);
assertTrue(indexFile.exists());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
