This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 9755a42a2 fix: validate RLBE decoder run lengths (#913)
9755a42a2 is described below
commit 9755a42a23275fe3eefeeee6ab7ca8b306511ada
Author: Colin Lee <[email protected]>
AuthorDate: Mon Aug 24 14:29:08 2026 +0800
fix: validate RLBE decoder run lengths (#913)
---
cpp/src/encoding/rlbe_decoder.h | 20 ++++-
cpp/test/encoding/rlbe_codec_test.cc | 79 +++++++++++++++++++
.../tsfile/encoding/decoder/IntRLBEDecoder.java | 20 ++++-
.../tsfile/encoding/decoder/LongRLBEDecoder.java | 20 ++++-
.../tsfile/encoding/decoder/RLBEDecoderTest.java | 91 ++++++++++++++++++++++
5 files changed, 226 insertions(+), 4 deletions(-)
diff --git a/cpp/src/encoding/rlbe_decoder.h b/cpp/src/encoding/rlbe_decoder.h
index f4028bc83..20337fa55 100644
--- a/cpp/src/encoding/rlbe_decoder.h
+++ b/cpp/src/encoding/rlbe_decoder.h
@@ -112,8 +112,8 @@ class RLBEDecoder : public Decoder {
return ret;
}
block_size_ = static_cast<int>(bits);
- if (block_size_ < 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) {
- return common::E_TSFILE_CORRUPTED;
+ if (block_size_ <= 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) {
+ return common::E_DECODE_ERR;
}
for (int i = 0; i < block_size_ * 2; ++i) {
data_[i] = 0;
@@ -136,6 +136,9 @@ class RLBEDecoder : public Decoder {
return ret;
}
int segment_length = static_cast<int>(bits);
+ if (segment_length < 1 || segment_length > Traits::VALUE_BITS) {
+ return common::E_DECODE_ERR;
+ }
int now = 0;
int next = 0;
@@ -147,10 +150,23 @@ class RLBEDecoder : public Decoder {
uint64_t run_length = 0;
int j = 1;
while (true) {
+ if (j >= static_cast<int>(sizeof(fibonacci_) /
+ sizeof(fibonacci_[0]))) {
+ return common::E_DECODE_ERR;
+ }
if (j > 1) {
fibonacci_[j] = fibonacci_[j - 1] + fibonacci_[j - 2];
+ if (fibonacci_[j] <= fibonacci_[j - 1]) {
+ return common::E_DECODE_ERR;
+ }
}
if (now == 1) {
+ const uint64_t remaining =
+ static_cast<uint64_t>(block_size_ - write_index_ - 1);
+ if (fibonacci_[j] > remaining ||
+ run_length > remaining - fibonacci_[j]) {
+ return common::E_DECODE_ERR;
+ }
run_length += fibonacci_[j];
}
if (now == 1 && next == 1) {
diff --git a/cpp/test/encoding/rlbe_codec_test.cc
b/cpp/test/encoding/rlbe_codec_test.cc
index f8571dc99..2955f7f47 100644
--- a/cpp/test/encoding/rlbe_codec_test.cc
+++ b/cpp/test/encoding/rlbe_codec_test.cc
@@ -20,6 +20,7 @@
#include <gtest/gtest.h>
#include <cmath>
+#include <cstdint>
#include <limits>
#include <vector>
@@ -30,6 +31,23 @@
namespace storage {
+namespace {
+
+void append_bits(std::vector<uint8_t>& bytes, int& bit_count, uint32_t value,
+ int width) {
+ for (int i = width - 1; i >= 0; --i) {
+ if (bytes.empty() || bit_count == 8) {
+ bytes.push_back(0);
+ bit_count = 0;
+ }
+ bytes.back() =
+ static_cast<uint8_t>((bytes.back() << 1) | ((value >> i) & 1));
+ ++bit_count;
+ }
+}
+
+} // namespace
+
TEST(RLBECodecTest, Int32RoundTrip) {
IntRLBEEncoder encoder;
IntRLBEDecoder decoder;
@@ -157,4 +175,65 @@ TEST(RLBECodecTest, FactoryAllocatesRLBECodecs) {
DecoderFactory::free(double_decoder);
}
+TEST(RLBECodecTest, RejectsInvalidBlockSize) {
+ std::vector<uint8_t> bytes(4, 0);
+ common::ByteStream stream;
+ stream.wrap_from(reinterpret_cast<const char*>(bytes.data()),
bytes.size());
+ IntRLBEDecoder decoder;
+ int32_t value = 0;
+ EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR);
+}
+
+TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) {
+ std::vector<uint8_t> bytes;
+ int bit_count = 0;
+ append_bits(bytes, bit_count, 1, 32); // block size
+ append_bits(bytes, bit_count, 1, 6); // segment length (int32 RLBE)
+ append_bits(bytes, bit_count, 0b011, 3); // Fibonacci code for run length
2
+ append_bits(bytes, bit_count, 0, 2); // delta payload (not reached)
+ bytes.back() <<= (8 - bit_count);
+
+ common::ByteStream stream;
+ stream.wrap_from(reinterpret_cast<const char*>(bytes.data()),
bytes.size());
+ IntRLBEDecoder decoder;
+ int32_t value = 0;
+ EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR);
+}
+
+TEST(RLBECodecTest, RejectsInvalidIntSegmentLength) {
+ for (int segment_length : {0, 33}) {
+ std::vector<uint8_t> bytes;
+ int bit_count = 0;
+ append_bits(bytes, bit_count, 1, 32); // block size
+ append_bits(bytes, bit_count, static_cast<uint32_t>(segment_length),
+ 6); // segment length
+ bytes.back() <<= (8 - bit_count);
+
+ common::ByteStream stream;
+ stream.wrap_from(reinterpret_cast<const char*>(bytes.data()),
+ bytes.size());
+ IntRLBEDecoder decoder;
+ int32_t value = 0;
+ EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR);
+ }
+}
+
+TEST(RLBECodecTest, RejectsInvalidLongSegmentLength) {
+ for (int segment_length : {0, 65}) {
+ std::vector<uint8_t> bytes;
+ int bit_count = 0;
+ append_bits(bytes, bit_count, 1, 32); // block size
+ append_bits(bytes, bit_count, static_cast<uint32_t>(segment_length),
+ 7); // segment length
+ bytes.back() <<= (8 - bit_count);
+
+ common::ByteStream stream;
+ stream.wrap_from(reinterpret_cast<const char*>(bytes.data()),
+ bytes.size());
+ LongRLBEDecoder decoder;
+ int64_t value = 0;
+ EXPECT_EQ(decoder.read_int64(value, stream), common::E_DECODE_ERR);
+ }
+}
+
} // namespace storage
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java
index ae46d3e0e..bd046e313 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java
@@ -19,11 +19,14 @@
package org.apache.tsfile.encoding.decoder;
+import org.apache.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import java.nio.ByteBuffer;
public class IntRLBEDecoder extends Decoder {
+ private static final int MAX_BLOCK_SIZE = 10000;
+
/** constructor of IntRLBEDecoder */
public IntRLBEDecoder() {
super(TSEncoding.RLBE);
@@ -65,6 +68,9 @@ public class IntRLBEDecoder extends Decoder {
readindex = -1;
clearBuffer(buffer);
readblocksize(buffer);
+ if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) {
+ throw new TsFileDecodingException("Invalid RLBE block size: " +
blocksize);
+ }
data = new int[blocksize * 2 + 1];
fibonacci = new int[blocksize * 2 + 1];
for (int i = 0; i < blocksize * 2; i++) {
@@ -89,6 +95,9 @@ public class IntRLBEDecoder extends Decoder {
for (int j = 5; j >= 0; j--) {
seglength |= (readbit(buffer) << j);
}
+ if (seglength < 1 || seglength > 32) {
+ throw new TsFileDecodingException("Invalid RLBE segment length: " +
seglength);
+ }
// generate repeat time of rle on delta
int now = readbit(buffer);
@@ -96,8 +105,17 @@ public class IntRLBEDecoder extends Decoder {
int j = 1;
while (true) {
+ if (j >= fibonacci.length) {
+ throw new TsFileDecodingException("Invalid RLBE Fibonacci run
length");
+ }
if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2];
- if (now == 1) runlength += fibonacci[j];
+ if (now == 1) {
+ long candidate = (long) runlength + fibonacci[j];
+ if (candidate <= 0 || candidate > blocksize - writeindex - 1) {
+ throw new TsFileDecodingException("Invalid RLBE run length: " +
candidate);
+ }
+ runlength = (int) candidate;
+ }
// when now and next are both 1, the 1 of next is the symbol of ending
of fibonacci code
if (now == 1 && next == 1) break;
j++;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java
index f41a5cd9b..6def046d8 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java
@@ -19,11 +19,14 @@
package org.apache.tsfile.encoding.decoder;
+import org.apache.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import java.nio.ByteBuffer;
public class LongRLBEDecoder extends Decoder {
+ private static final int MAX_BLOCK_SIZE = 10000;
+
/** constructor of LongRLBEDecoder */
public LongRLBEDecoder() {
super(TSEncoding.RLBE);
@@ -65,6 +68,9 @@ public class LongRLBEDecoder extends Decoder {
readindex = -1;
clearBuffer(buffer);
readblocksize(buffer);
+ if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) {
+ throw new TsFileDecodingException("Invalid RLBE block size: " +
blocksize);
+ }
data = new long[blocksize * 2 + 1];
fibonacci = new long[blocksize * 2 + 1];
for (int i = 0; i < blocksize * 2; i++) {
@@ -90,6 +96,9 @@ public class LongRLBEDecoder extends Decoder {
for (int j = 6; j >= 0; j--) {
seglength |= (readbit(buffer) << j);
}
+ if (seglength < 1 || seglength > 64) {
+ throw new TsFileDecodingException("Invalid RLBE segment length: " +
seglength);
+ }
// generate repeat time of rle on delta
int now = readbit(buffer);
@@ -97,8 +106,17 @@ public class LongRLBEDecoder extends Decoder {
int j = 1;
while (true) {
+ if (j >= fibonacci.length) {
+ throw new TsFileDecodingException("Invalid RLBE Fibonacci run
length");
+ }
if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2];
- if (now == 1) runlength += fibonacci[j];
+ if (now == 1) {
+ long value = fibonacci[j];
+ if (value <= 0 || runlength > blocksize - writeindex - 1 - value) {
+ throw new TsFileDecodingException("Invalid RLBE run length");
+ }
+ runlength += value;
+ }
// when now and next are both 1, the 1 of next is the symbol of ending
of fibonacci code
if (now == 1 && next == 1) break;
j++;
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java
index bdb8082ee..db97cc25c 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java
@@ -23,6 +23,7 @@ import org.apache.tsfile.encoding.encoder.Encoder;
import org.apache.tsfile.encoding.encoder.FloatRLBE;
import org.apache.tsfile.encoding.encoder.TSEncodingBuilder;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.junit.After;
@@ -258,6 +259,96 @@ public class RLBEDecoderTest {
assertEquals(f, decoder.readDouble(buffer), doubleDelta);
}
+ @Test
+ public void testRejectsInvalidBlockSize() {
+ ByteBuffer buffer = ByteBuffer.wrap(new byte[4]);
+ try {
+ new IntRLBEDecoder().readInt(buffer);
+ fail("Expected invalid RLBE block size to be rejected");
+ } catch (TsFileDecodingException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testRejectsRunLengthBeyondBlock() {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ BitWriter bits = new BitWriter(output);
+ bits.write(1, 32); // block size
+ bits.write(1, 6); // segment length
+ bits.write(0b011, 3); // Fibonacci code for run length 2, plus terminator
+ bits.write(0, 2); // delta payload (not reached after validation)
+ bits.flush();
+
+ try {
+ new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray()));
+ fail("Expected RLBE run length to be rejected");
+ } catch (TsFileDecodingException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testRejectsInvalidIntSegmentLength() {
+ for (int seglength : new int[] {0, 33}) {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ BitWriter bits = new BitWriter(output);
+ bits.write(1, 32); // block size
+ bits.write(seglength, 6); // invalid segment length
+ bits.flush();
+ try {
+ new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray()));
+ fail("Expected RLBE segment length to be rejected: " + seglength);
+ } catch (TsFileDecodingException expected) {
+ // expected
+ }
+ }
+ }
+
+ @Test
+ public void testRejectsInvalidLongSegmentLength() {
+ for (int seglength : new int[] {0, 65}) {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ BitWriter bits = new BitWriter(output);
+ bits.write(1, 32); // block size
+ bits.write(seglength, 7); // invalid segment length
+ bits.flush();
+ try {
+ new LongRLBEDecoder().readLong(ByteBuffer.wrap(output.toByteArray()));
+ fail("Expected RLBE segment length to be rejected: " + seglength);
+ } catch (TsFileDecodingException expected) {
+ // expected
+ }
+ }
+ }
+
+ private static class BitWriter {
+ private final ByteArrayOutputStream output;
+ private int currentByte;
+ private int bitCount;
+
+ private BitWriter(ByteArrayOutputStream output) {
+ this.output = output;
+ }
+
+ private void write(int value, int width) {
+ for (int i = width - 1; i >= 0; i--) {
+ currentByte = (currentByte << 1) | ((value >>> i) & 1);
+ if (++bitCount == 8) {
+ output.write(currentByte);
+ currentByte = 0;
+ bitCount = 0;
+ }
+ }
+ }
+
+ private void flush() {
+ if (bitCount > 0) {
+ output.write(currentByte << (8 - bitCount));
+ }
+ }
+ }
+
private void testFloatLength(List<Float> valueList, boolean isDebug, int
repeatCount)
throws Exception {
Encoder encoder = new FloatRLBE();
