This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dd430bd2497 [Session] Fix tablet compression overflow with GZIP and
LZMA2 (#19179)
dd430bd2497 is described below
commit dd430bd2497d6cacfb5bb7414f81bf5bf1c57a05
Author: Jackie Tien <[email protected]>
AuthorDate: Mon Nov 9 08:28:06 2026 +0800
[Session] Fix tablet compression overflow with GZIP and LZMA2 (#19179)
---
.../iotdb/session/rpccompress/TabletEncoder.java | 23 ++-
.../rpccompress/TabletEncoderCompressionTest.java | 230 +++++++++++++++++++++
2 files changed, 246 insertions(+), 7 deletions(-)
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
index 53964ec0ae8..9ec88ab3210 100644
---
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
+++
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
@@ -37,6 +37,7 @@ import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.List;
public class TabletEncoder {
@@ -140,14 +141,22 @@ public class TabletEncoder {
if (compressionType != CompressionType.UNCOMPRESSED) {
ICompressor compressor = ICompressor.getCompressor(compressionType);
int uncompressedSize = buffer.remaining();
- byte[] compressed = new
byte[compressor.getMaxBytesForCompression(uncompressedSize) + 4];
+ int offset = buffer.arrayOffset() + buffer.position();
try {
- int compressedLength =
- compressor.compress(
- buffer.array(),
- buffer.arrayOffset() + buffer.position(),
- uncompressedSize,
- compressed);
+ byte[] compressed;
+ int compressedLength;
+ // getMaxBytesForCompression is a real bound except for GZIP and
LZMA2, whose output can
+ // outgrow it on data that hardly compresses. Those two size their own
output instead; the
+ // others write straight into one buffer, which saves an array and a
copy.
+ if (compressionType == CompressionType.GZIP || compressionType ==
CompressionType.LZMA2) {
+ byte[] output = compressor.compress(buffer.array(), offset,
uncompressedSize);
+ compressedLength = output.length;
+ compressed = Arrays.copyOf(output, compressedLength + 4);
+ } else {
+ compressed = new
byte[compressor.getMaxBytesForCompression(uncompressedSize) + 4];
+ compressedLength =
+ compressor.compress(buffer.array(), offset, uncompressedSize,
compressed);
+ }
buffer = ByteBuffer.wrap(compressed, 0, compressedLength + 4);
buffer.position(compressedLength);
buffer.putInt(uncompressedSize);
diff --git
a/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderCompressionTest.java
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderCompressionTest.java
new file mode 100644
index 00000000000..a8757e098c7
--- /dev/null
+++
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderCompressionTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.rpccompress;
+
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.encoding.decoder.Decoder;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Function;
+
+/**
+ * {@link TabletEncoder} must compress data that does not shrink, whatever
{@link
+ * ICompressor#getMaxBytesForCompression} estimates for it, and keep writing
the format the DataNode
+ * reads: the compressed bytes followed by the uncompressed length as a
big-endian int.
+ */
+public class TabletEncoderCompressionTest {
+
+ private static final List<TSEncoding> ALL_PLAIN =
+ Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN);
+
+ @Test
+ public void gzipCompressesNoisyDoublesEncodedWithGorilla() throws
IOException {
+ int rows = 10;
+ Tablet tablet = tablet(TSDataType.DOUBLE, rows);
+ Random random = new Random(1);
+ double measurement = 20.0;
+ double[] expected = new double[rows];
+ for (int row = 0; row < rows; row++) {
+ // a slowly drifting, noisy measurement
+ measurement += (random.nextDouble() - 0.5) * 0.1;
+ expected[row] = measurement + random.nextGaussian() * 0.01;
+ tablet.addTimestamp(row, row);
+ tablet.addValue(row, 0, expected[row]);
+ }
+ List<TSEncoding> encodings = Arrays.asList(TSEncoding.TS_2DIFF,
TSEncoding.GORILLA);
+
+ ByteBuffer uncompressed =
+ roundTrip(CompressionType.GZIP, encodings, encoder ->
encoder.encodeValues(tablet));
+
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.GORILLA,
TSDataType.DOUBLE);
+ for (int row = 0; row < rows; row++) {
+ Assert.assertEquals(expected[row], decoder.readDouble(uncompressed), 0);
+ }
+ }
+
+ @Test
+ public void gzipCompressesRandomLongsEncodedWithPlain() throws IOException {
+ int rows = 1000;
+ Tablet tablet = randomLongTablet(rows, new Random(2));
+
+ ByteBuffer uncompressed =
+ roundTrip(CompressionType.GZIP, ALL_PLAIN, encoder ->
encoder.encodeValues(tablet));
+
+ long[] expected = (long[]) tablet.getValues()[0];
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.PLAIN,
TSDataType.INT64);
+ for (int row = 0; row < rows; row++) {
+ Assert.assertEquals(expected[row], decoder.readLong(uncompressed));
+ }
+ }
+
+ @Test
+ public void gzipCompressesRandomTimestampsEncodedWithPlain() throws
IOException {
+ int rows = 1000;
+ Tablet tablet = randomLongTablet(rows, new Random(3));
+
+ ByteBuffer uncompressed =
+ roundTrip(CompressionType.GZIP, ALL_PLAIN, encoder ->
encoder.encodeTime(tablet));
+
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.PLAIN,
TSDataType.INT64);
+ for (int row = 0; row < rows; row++) {
+ Assert.assertEquals(tablet.getTimestamp(row),
decoder.readLong(uncompressed));
+ }
+ }
+
+ /**
+ * Walks across the sizes where the GZIP output first outgrows the estimate.
Output that is at
+ * most 4 bytes over it still fits an array with room for the length, but
not together with the
+ * length, so this covers both ways of running out of room.
+ */
+ @Test
+ public void gzipCompressesIncompressibleDataOfEverySmallSize() throws
IOException {
+ Random random = new Random(4);
+ for (int rows = 1; rows <= 64; rows++) {
+ Tablet tablet = randomLongTablet(rows, random);
+ roundTrip(CompressionType.GZIP, ALL_PLAIN, encoder ->
encoder.encodeValues(tablet));
+ }
+ }
+
+ /** LZMA2 only estimates 100 extra bytes, while its overhead grows with the
input. */
+ @Test
+ public void lzma2CompressesLargeIncompressibleData() throws IOException {
+ Tablet tablet = randomLongTablet(150_000, new Random(5));
+ roundTrip(CompressionType.LZMA2, ALL_PLAIN, encoder ->
encoder.encodeValues(tablet));
+ }
+
+ @Test
+ public void everyCodecRoundTripsIncompressibleAndCompressibleData() throws
IOException {
+ for (CompressionType type :
+ Arrays.asList(
+ CompressionType.SNAPPY,
+ CompressionType.LZ4,
+ CompressionType.GZIP,
+ CompressionType.ZSTD,
+ CompressionType.LZMA2)) {
+ Tablet incompressible = randomLongTablet(1000, new Random(6));
+ roundTrip(type, ALL_PLAIN, encoder ->
encoder.encodeValues(incompressible));
+ roundTrip(type, ALL_PLAIN, encoder ->
encoder.encodeTime(incompressible));
+
+ Tablet compressible = repeatedLongTablet(1000);
+ roundTrip(type, ALL_PLAIN, encoder ->
encoder.encodeValues(compressible));
+ roundTrip(type, ALL_PLAIN, encoder -> encoder.encodeTime(compressible));
+ }
+ }
+
+ /** Pins the format on data for which the estimate has always been large
enough. */
+ @Test
+ public void compressedBytesStayTheSameAsWithEstimateSizedBuffer() throws
IOException {
+ for (CompressionType type :
+ Arrays.asList(CompressionType.SNAPPY, CompressionType.LZ4,
CompressionType.ZSTD)) {
+ assertSameAsEstimateSizedCompression(type, randomLongTablet(1000, new
Random(7)));
+ assertSameAsEstimateSizedCompression(type, repeatedLongTablet(1000));
+ }
+ assertSameAsEstimateSizedCompression(CompressionType.GZIP,
repeatedLongTablet(1000));
+ assertSameAsEstimateSizedCompression(CompressionType.LZMA2,
repeatedLongTablet(1000));
+ }
+
+ private static void assertSameAsEstimateSizedCompression(CompressionType
type, Tablet tablet)
+ throws IOException {
+ byte[] raw =
+ sentBytes(new TabletEncoder(CompressionType.UNCOMPRESSED,
ALL_PLAIN).encodeValues(tablet));
+ ICompressor compressor = ICompressor.getCompressor(type);
+ byte[] compressed = new
byte[compressor.getMaxBytesForCompression(raw.length) + 4];
+ int compressedLength = compressor.compress(raw, 0, raw.length, compressed);
+ ByteBuffer.wrap(compressed, compressedLength, 4).putInt(raw.length);
+
+ Assert.assertArrayEquals(
+ type.toString(),
+ Arrays.copyOf(compressed, compressedLength + 4),
+ sentBytes(new TabletEncoder(type, ALL_PLAIN).encodeValues(tablet)));
+ }
+
+ /**
+ * Encodes a column with the given codec, decompresses it the way {@code
TabletDecoder} does on
+ * the DataNode, and checks that this gives back the uncompressed encoding
of the column.
+ *
+ * @return the decompressed column
+ */
+ private static ByteBuffer roundTrip(
+ CompressionType type, List<TSEncoding> encodings,
Function<TabletEncoder, ByteBuffer> column)
+ throws IOException {
+ byte[] raw =
+ sentBytes(column.apply(new TabletEncoder(CompressionType.UNCOMPRESSED,
encodings)));
+ ByteBuffer received =
+ ByteBuffer.wrap(sentBytes(column.apply(new TabletEncoder(type,
encodings))));
+
+ int uncompressedLength = received.getInt(received.limit() - 4);
+ byte[] uncompressed = new byte[uncompressedLength];
+ IUnCompressor.getUnCompressor(type)
+ .uncompress(
+ received.array(),
+ received.arrayOffset() + received.position(),
+ received.remaining() - 4,
+ uncompressed,
+ 0);
+
+ Assert.assertArrayEquals(type.toString(), raw, uncompressed);
+ return ByteBuffer.wrap(uncompressed);
+ }
+
+ /** Thrift sends what is between the position and the limit of the buffer. */
+ private static byte[] sentBytes(ByteBuffer buffer) {
+ byte[] bytes = new byte[buffer.remaining()];
+ buffer.duplicate().get(bytes);
+ return bytes;
+ }
+
+ private static Tablet randomLongTablet(int rows, Random random) {
+ Tablet tablet = tablet(TSDataType.INT64, rows);
+ for (int row = 0; row < rows; row++) {
+ tablet.addTimestamp(row, random.nextLong());
+ tablet.addValue(row, 0, random.nextLong());
+ }
+ return tablet;
+ }
+
+ private static Tablet repeatedLongTablet(int rows) {
+ Tablet tablet = tablet(TSDataType.INT64, rows);
+ for (int row = 0; row < rows; row++) {
+ tablet.addTimestamp(row, 1_700_000_000_000L);
+ tablet.addValue(row, 0, 42L);
+ }
+ return tablet;
+ }
+
+ private static Tablet tablet(TSDataType type, int rows) {
+ List<IMeasurementSchema> schemas = Collections.singletonList(new
MeasurementSchema("s0", type));
+ return new Tablet("root.sg.d1", schemas, rows);
+ }
+}