This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch xiangfu0/codex/codec-stack/02-runtime
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to
refs/heads/xiangfu0/codex/codec-stack/02-runtime by this push:
new 7c5dfae64c9 Address codec runtime review nits
7c5dfae64c9 is described below
commit 7c5dfae64c9af6315a9ae5991e045a72bcb6d53a
Author: Xiang Fu <[email protected]>
AuthorDate: Mon Aug 24 18:35:26 2026 -0700
Address codec runtime review nits
---
.../segment/local/io/codec/CodecBufferUtils.java | 6 +++---
.../pinot/segment/local/io/codec/CodecContext.java | 3 +--
.../local/io/codec/CodecPipelineExecutor.java | 2 +-
.../segment/local/io/codec/ZstdCodecDefinition.java | 6 ++++--
.../io/codec/CompressionCodecCorruptInputTest.java | 20 ++++++++++++++++++++
5 files changed, 29 insertions(+), 8 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecBufferUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecBufferUtils.java
index 30c5f50911d..057c574e98a 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecBufferUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecBufferUtils.java
@@ -26,14 +26,14 @@ import org.apache.pinot.segment.spi.memory.CleanerUtil;
/// Package-private buffer helpers shared across codec handler implementations.
final class CodecBufferUtils {
+ private CodecBufferUtils() {
+ }
+
/// Sanity cap (1 GiB) on any decompressed size declared by untrusted
encoded segment data. A
/// corrupt or hostile declaration must never drive a giant pre-allocation;
1 GiB is well above
/// any realistic chunk size.
static final long MAX_DECLARED_DECOMPRESSED_SIZE = 1L << 30;
- private CodecBufferUtils() {
- }
-
/// Validates a decompressed size declared by untrusted segment data and
returns it when in range.
///
/// Shared by every codec definition so the bound cannot be omitted from a
new codec or silently
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecContext.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecContext.java
index 0d813b3cc9f..011f6d958f4 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecContext.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecContext.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.io.codec;
-import java.util.Objects;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -30,7 +29,7 @@ final class CodecContext {
private final DataType _dataType;
CodecContext(DataType dataType) {
- _dataType = Objects.requireNonNull(dataType, "dataType");
+ _dataType = dataType;
}
/// Returns the stored [DataType] of the column being indexed.
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecPipelineExecutor.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecPipelineExecutor.java
index fb6657725df..4e4c8b56877 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecPipelineExecutor.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/CodecPipelineExecutor.java
@@ -357,7 +357,7 @@ public final class CodecPipelineExecutor {
public void decode(ByteBuffer src, ByteBuffer dst, int expectedDecodedSize,
int maxIntermediateSize,
long maxCumulativeSize, DecodeScratch scratch)
throws IOException {
- Preconditions.checkNotNull(scratch, "scratch").ensureOpen();
+ scratch.ensureOpen();
Preconditions.checkArgument(!_requiresDirectDstBuffer || dst.isDirect(),
"decode(src, dst) requires a direct ByteBuffer for pipeline: %s",
_canonicalSpec);
Preconditions.checkArgument(expectedDecodedSize >= 0 &&
expectedDecodedSize <= dst.capacity(),
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/ZstdCodecDefinition.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/ZstdCodecDefinition.java
index 03caa98af48..559b39475c4 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/ZstdCodecDefinition.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/codec/ZstdCodecDefinition.java
@@ -196,11 +196,13 @@ final class ZstdCodecDefinition implements
ChunkCodecHandler<ZstdCodecDefinition
dst.clear();
ByteBuffer directSrc = CodecBufferUtils.toDirectBuffer(src);
try {
- long decompressedSize = Zstd.getFrameContentSize(directSrc);
+ long declaredDecompressedSize = Zstd.getFrameContentSize(directSrc);
// As in decode(), zero is a valid known size; only Zstd's negative
sentinel values are errors.
- if (decompressedSize < 0) {
+ if (declaredDecompressedSize < 0) {
throw new IOException("Zstd: cannot determine decompressed size from
frame header");
}
+ int decompressedSize = CodecBufferUtils.checkDeclaredDecompressedSize(
+ declaredDecompressedSize, "Zstd", "frame header");
if (decompressedSize > dst.capacity()) {
throw new IllegalArgumentException(
"Zstd: decompressed size " + decompressedSize + " exceeds dst
capacity " + dst.capacity());
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/codec/CompressionCodecCorruptInputTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/codec/CompressionCodecCorruptInputTest.java
index 7b39cccc60a..99a0e2a0914 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/codec/CompressionCodecCorruptInputTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/codec/CompressionCodecCorruptInputTest.java
@@ -61,6 +61,26 @@ public class CompressionCodecCorruptInputTest {
garbage(64)));
}
+ @Test
+ public void testZstdDecodeIntoRejectsOversizedDeclaredSize() {
+ // decodeInto must apply the shared sanity cap before its later
destination-capacity check.
+ ByteBuffer dst = ByteBuffer.allocateDirect(16);
+ IOException exception = expectThrows(IOException.class,
+ () -> ZstdCodecDefinition.INSTANCE.decodeInto(
+ ZstdCodecDefinition.INSTANCE.parseOptions(List.of()), CTX,
oversizedZstdFrame(), dst));
+ assertTrue(exception.getMessage().contains("out of range"),
exception.getMessage());
+ }
+
+ /// Frame header whose content-size field declares (1 << 30) + 1
decompressed bytes.
+ private static ByteBuffer oversizedZstdFrame() {
+ ByteBuffer frame = ByteBuffer.allocateDirect(9);
+ frame.put(new byte[]{
+ 0x28, (byte) 0xB5, 0x2F, (byte) 0xFD, (byte) 0xA0, 0x01, 0x00, 0x00,
0x40
+ });
+ frame.flip();
+ return frame;
+ }
+
@Test
public void testSnappyDecodeRejectsGarbage() {
assertThrows(Exception.class,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
