This is an automated email from the ASF dual-hosted git repository.

JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new c46fe4b8490 [Session] Write PLAIN-encoded tablet columns in bulk 
instead of byte by byte (#19176)
c46fe4b8490 is described below

commit c46fe4b84906ca8720dd13b319ec9dbe716c2f71
Author: Christofer Dutz <[email protected]>
AuthorDate: Sun Nov 8 14:09:55 2026 +0100

    [Session] Write PLAIN-encoded tablet columns in bulk instead of byte by 
byte (#19176)
---
 .../iotdb/session/rpccompress/TabletEncoder.java   |  81 ++----
 .../iotdb/session/util/SessionTypeServices.java    | 141 ++++++++++
 .../apache/iotdb/session/util/SessionUtils.java    |  17 ++
 .../rpccompress/TabletEncoderPlainTest.java        | 289 +++++++++++++++++++++
 4 files changed, 475 insertions(+), 53 deletions(-)

diff --git 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
index be44c5feb20..53964ec0ae8 100644
--- 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
+++ 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
@@ -27,12 +27,9 @@ import org.apache.tsfile.encoding.encoder.TSEncodingBuilder;
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.file.metadata.enums.CompressionType;
 import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.utils.Binary;
 import org.apache.tsfile.utils.BitMap;
 import org.apache.tsfile.utils.BytesUtils;
-import org.apache.tsfile.utils.DateUtils;
 import org.apache.tsfile.utils.PublicBAOS;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
 import org.apache.tsfile.write.record.Tablet;
 import org.apache.tsfile.write.schema.IMeasurementSchema;
 
@@ -40,7 +37,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.time.LocalDate;
 import java.util.List;
 
 public class TabletEncoder {
@@ -57,6 +53,10 @@ public class TabletEncoder {
   }
 
   public ByteBuffer encodeTime(Tablet tablet) {
+    if (encodingList.get(0) == TSEncoding.PLAIN) {
+      // the same bytes as PlainEncoder (big-endian longs), written in one go
+      return compressBuffer(SessionUtils.getTimeBuffer(tablet));
+    }
     Encoder encoder =
         
TSEncodingBuilder.getEncodingBuilder(encodingList.get(0)).getEncoder(TSDataType.INT64);
     PublicBAOS baos = new PublicBAOS();
@@ -74,25 +74,16 @@ public class TabletEncoder {
   }
 
   public ByteBuffer encodeValues(Tablet tablet) {
-    PublicBAOS baos = new PublicBAOS();
     List<IMeasurementSchema> schemas = tablet.getSchemas();
+    PublicBAOS baos = newValueStream(tablet, schemas);
     for (int j = 0, schemasSize = schemas.size(); j < schemasSize; j++) {
-      IMeasurementSchema schema = schemas.get(j);
-      TSDataType dataType = schema.getType();
+      TSDataType dataType = schemas.get(j).getType();
       TSEncoding encoding = encodingList.get(j + 1);
-      Encoder encoder = 
TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
       if (encoding == TSEncoding.PLAIN) {
-        // PlainEncoder uses var int, which may cause compatibility problem
-        if (dataType.isBinary()) {
-          encodePlainBinary(tablet, j, baos);
-        } else if (dataType == TSDataType.INT32) {
-          encodePlainI32(tablet, j, baos);
-        } else if (dataType == TSDataType.DATE) {
-          encodePlainDate(tablet, j, baos);
-        } else {
-          SessionUtils.encodeValue(dataType, tablet, j, encoder, baos);
-        }
+        // not through PlainEncoder, which writes INT32 as a var int
+        SessionUtils.encodePlainValue(dataType, tablet, j, baos);
       } else {
+        Encoder encoder = 
TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
         SessionUtils.encodeValue(dataType, tablet, j, encoder, baos);
       }
     }
@@ -103,45 +94,29 @@ public class TabletEncoder {
     return compressBuffer(buffer);
   }
 
-  private void encodePlainBinary(Tablet tablet, int j, ByteArrayOutputStream 
baos) {
-    Binary[] binaryValues = (Binary[]) tablet.getValues()[j];
-    try {
-      for (int index = 0; index < tablet.getRowSize(); index++) {
-        if (!tablet.isNull(index, j)) {
-          ReadWriteIOUtils.write(binaryValues[index], baos);
-        } else {
-          ReadWriteIOUtils.write(Binary.EMPTY_VALUE, baos);
-        }
+  /**
+   * Pre-sizes the stream for the PLAIN columns, whose size is known up front, 
and the bit maps
+   * after them, so that the stream does not have to double for the last few 
bytes. Other encodings
+   * and binary values let the stream grow, as does a size that does not fit 
in an array.
+   */
+  private PublicBAOS newValueStream(Tablet tablet, List<IMeasurementSchema> 
schemas) {
+    int rows = tablet.getRowSize();
+    long size = 0;
+    for (int j = 0, schemasSize = schemas.size(); j < schemasSize; j++) {
+      if (encodingList.get(j + 1) == TSEncoding.PLAIN) {
+        size += (long) rows * 
SessionUtils.getPlainRowSize(schemas.get(j).getType());
       }
-    } catch (IOException e) {
-      throw new IllegalStateException(e);
     }
-  }
-
-  private void encodePlainI32(Tablet tablet, int j, ByteArrayOutputStream 
baos) {
-    int[] intValues = (int[]) tablet.getValues()[j];
-    try {
-      for (int index = 0; index < tablet.getRowSize(); index++) {
-        ReadWriteIOUtils.write(intValues[index], baos);
-      }
-    } catch (IOException e) {
-      throw new IllegalStateException(e);
+    if (size == 0) {
+      return new PublicBAOS();
     }
-  }
-
-  private void encodePlainDate(Tablet tablet, int j, ByteArrayOutputStream 
baos) {
-    LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[j];
-    try {
-      for (int index = 0; index < tablet.getRowSize(); index++) {
-        if (!tablet.isNull(index, j)) {
-          
ReadWriteIOUtils.write(DateUtils.parseDateExpressionToInt(dateValues[index]), 
baos);
-        } else {
-          ReadWriteIOUtils.write(0, baos);
-        }
-      }
-    } catch (IOException e) {
-      throw new IllegalStateException(e);
+    BitMap[] bitMaps = tablet.getBitMaps();
+    if (bitMaps != null) {
+      // at most a marker byte and the truncated bit map per column
+      size += bitMaps.length * (1L + rows / Byte.SIZE + 1);
     }
+    // stay clear of the VM's array size limit
+    return size <= Integer.MAX_VALUE - 8 ? new PublicBAOS((int) size) : new 
PublicBAOS();
   }
 
   private void encodeBitMap(Tablet tablet, ByteArrayOutputStream baos) {
diff --git 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
index 0daca161a60..d4b26820fcb 100644
--- 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
+++ 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
@@ -35,6 +35,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException;
 import org.apache.tsfile.write.record.Tablet;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.LocalDate;
 
@@ -276,6 +277,127 @@ final class SessionTypeServices {
                 };
           };
 
+  /**
+   * Writes a tablet column in the PLAIN layout that {@code TabletEncoder} 
sends. The fixed-width
+   * types produce the same bytes as {@code PlainEncoder} through {@link
+   * #TABLET_VALUE_ENCODER_SERVICE} (big-endian; a null repeats the last 
non-null value, 0 before
+   * the first one), but fill one buffer per column instead of making one 
{@code write(int)} call,
+   * with its capacity check, per byte. INT32, DATE and the binary types keep 
their fixed-length
+   * layout, because {@code PlainEncoder} writes INT32 as a var int.
+   */
+  private static final TypeService<TabletPlainValueWriter> 
TABLET_PLAIN_VALUE_WRITER_SERVICE =
+      type ->
+          switch (type.getTypeEnum()) {
+            case INT32 ->
+                (tablet, columnIndex, outputStream) -> {
+                  int[] values = (int[]) tablet.getValues()[columnIndex];
+                  ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize() 
* Integer.BYTES);
+                  for (int index = 0; index < tablet.getRowSize(); index++) {
+                    buffer.putInt(values[index]);
+                  }
+                  writeTo(buffer, outputStream);
+                };
+            case DATE ->
+                (tablet, columnIndex, outputStream) -> {
+                  LocalDate[] values = (LocalDate[]) 
tablet.getValues()[columnIndex];
+                  ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize() 
* Integer.BYTES);
+                  for (int index = 0; index < tablet.getRowSize(); index++) {
+                    buffer.putInt(
+                        tablet.isNull(index, columnIndex)
+                            ? 0
+                            : 
DateUtils.parseDateExpressionToInt(values[index]));
+                  }
+                  writeTo(buffer, outputStream);
+                };
+            case INT64, TIMESTAMP ->
+                (tablet, columnIndex, outputStream) -> {
+                  long[] values = (long[]) tablet.getValues()[columnIndex];
+                  ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize() 
* Long.BYTES);
+                  long lastNonNullValue = 0;
+                  for (int index = 0; index < tablet.getRowSize(); index++) {
+                    if (!tablet.isNull(index, columnIndex)) {
+                      lastNonNullValue = values[index];
+                    }
+                    buffer.putLong(lastNonNullValue);
+                  }
+                  writeTo(buffer, outputStream);
+                };
+            case FLOAT ->
+                (tablet, columnIndex, outputStream) -> {
+                  float[] values = (float[]) tablet.getValues()[columnIndex];
+                  ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize() 
* Float.BYTES);
+                  float lastNonNullValue = 0.0f;
+                  for (int index = 0; index < tablet.getRowSize(); index++) {
+                    if (!tablet.isNull(index, columnIndex)) {
+                      lastNonNullValue = values[index];
+                    }
+                    // floatToIntBits like PlainEncoder, which collapses NaN 
payloads
+                    buffer.putInt(Float.floatToIntBits(lastNonNullValue));
+                  }
+                  writeTo(buffer, outputStream);
+                };
+            case DOUBLE ->
+                (tablet, columnIndex, outputStream) -> {
+                  double[] values = (double[]) tablet.getValues()[columnIndex];
+                  ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize() 
* Double.BYTES);
+                  double lastNonNullValue = 0.0;
+                  for (int index = 0; index < tablet.getRowSize(); index++) {
+                    if (!tablet.isNull(index, columnIndex)) {
+                      lastNonNullValue = values[index];
+                    }
+                    // doubleToLongBits like PlainEncoder, which collapses NaN 
payloads
+                    buffer.putLong(Double.doubleToLongBits(lastNonNullValue));
+                  }
+                  writeTo(buffer, outputStream);
+                };
+            case BOOLEAN ->
+                (tablet, columnIndex, outputStream) -> {
+                  boolean[] values = (boolean[]) 
tablet.getValues()[columnIndex];
+                  byte[] bytes = new byte[tablet.getRowSize()];
+                  boolean lastNonNullValue = false;
+                  for (int index = 0; index < bytes.length; index++) {
+                    if (!tablet.isNull(index, columnIndex)) {
+                      lastNonNullValue = values[index];
+                    }
+                    bytes[index] = BytesUtils.boolToByte(lastNonNullValue);
+                  }
+                  outputStream.write(bytes, 0, bytes.length);
+                };
+            case TEXT, STRING, BLOB, OBJECT ->
+                (tablet, columnIndex, outputStream) -> {
+                  Binary[] values = (Binary[]) tablet.getValues()[columnIndex];
+                  try {
+                    for (int index = 0; index < tablet.getRowSize(); index++) {
+                      ReadWriteIOUtils.write(
+                          tablet.isNull(index, columnIndex) ? 
Binary.EMPTY_VALUE : values[index],
+                          outputStream);
+                    }
+                  } catch (IOException e) {
+                    throw new IllegalStateException(e);
+                  }
+                };
+            case ROW, UNKNOWN, VECTOR ->
+                (tablet, columnIndex, outputStream) -> {
+                  throw unsupportedTabletDataType(type.getTypeEnum());
+                };
+          };
+
+  /**
+   * Bytes per row of a column in the PLAIN layout of {@link 
#TABLET_PLAIN_VALUE_WRITER_SERVICE}, to
+   * pre-size the output. Binary types count only their length prefix, so that 
no values are read;
+   * the stream grows for their bytes.
+   */
+  private static final TypeService<Integer> TABLET_PLAIN_ROW_SIZE_SERVICE =
+      type ->
+          switch (type.getTypeEnum()) {
+            case BOOLEAN -> Byte.BYTES;
+            case INT32, FLOAT, DATE -> Integer.BYTES;
+            case INT64, DOUBLE, TIMESTAMP -> Long.BYTES;
+            case TEXT, STRING, BLOB, OBJECT -> Integer.BYTES;
+            // the writer refuses these types
+            case ROW, UNKNOWN, VECTOR -> 0;
+          };
+
   private static final TypeService<ValueListSorter> VALUE_LIST_SORTER_SERVICE =
       type ->
           switch (type.getTypeEnum()) {
@@ -354,6 +476,8 @@ final class SessionTypeServices {
     TABLET_COLUMN_OCCUPATION_CALCULATOR_SERVICE.check();
     TABLET_VALUE_WRITER_SERVICE.check();
     TABLET_VALUE_ENCODER_SERVICE.check();
+    TABLET_PLAIN_VALUE_WRITER_SERVICE.check();
+    TABLET_PLAIN_ROW_SIZE_SERVICE.check();
     VALUE_LIST_SORTER_SERVICE.check();
   }
 
@@ -379,6 +503,14 @@ final class SessionTypeServices {
     return TABLET_VALUE_ENCODER_SERVICE.call(Type.fromTsDataType(dataType));
   }
 
+  static TabletPlainValueWriter tabletPlainValueWriter(TSDataType dataType) {
+    return 
TABLET_PLAIN_VALUE_WRITER_SERVICE.call(Type.fromTsDataType(dataType));
+  }
+
+  static int tabletPlainRowSize(TSDataType dataType) {
+    return TABLET_PLAIN_ROW_SIZE_SERVICE.call(Type.fromTsDataType(dataType));
+  }
+
   static ValueListSorter valueListSorter(TSDataType dataType) {
     return VALUE_LIST_SORTER_SERVICE.call(Type.fromTsDataType(dataType));
   }
@@ -390,6 +522,10 @@ final class SessionTypeServices {
     return ((String) value).getBytes(TSFileConfig.STRING_CHARSET);
   }
 
+  private static void writeTo(ByteBuffer buffer, ByteArrayOutputStream 
outputStream) {
+    outputStream.write(buffer.array(), 0, buffer.position());
+  }
+
   private static IoTDBConnectionException unsupportedDataType(Object dataType) 
{
     return new IoTDBConnectionException(Session.MSG_UNSUPPORTED_DATA_TYPE + 
dataType);
   }
@@ -429,6 +565,11 @@ final class SessionTypeServices {
         Tablet tablet, int columnIndex, Encoder encoder, ByteArrayOutputStream 
outputStream);
   }
 
+  @FunctionalInterface
+  interface TabletPlainValueWriter {
+    void write(Tablet tablet, int columnIndex, ByteArrayOutputStream 
outputStream);
+  }
+
   @FunctionalInterface
   interface ValueListSorter {
     Object sort(Object valueList, int[] index);
diff --git 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
index 5203fffd915..2692f1c600a 100644
--- 
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
+++ 
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
@@ -193,6 +193,23 @@ public class SessionUtils {
     }
   }
 
+  /**
+   * Writes a tablet column in the PLAIN layout of the RPC tablet compression: 
fixed-width values
+   * big-endian (INT32 not as a var int), binary values with an int length 
prefix.
+   */
+  public static void encodePlainValue(
+      TSDataType dataType, Tablet tablet, int i, ByteArrayOutputStream 
outputStream) {
+    SessionTypeServices.tabletPlainValueWriter(dataType).write(tablet, i, 
outputStream);
+  }
+
+  /**
+   * @return the bytes per row of a column in the layout of {@link 
#encodePlainValue}, without the
+   *     bytes of binary values
+   */
+  public static int getPlainRowSize(TSDataType dataType) {
+    return SessionTypeServices.tabletPlainRowSize(dataType);
+  }
+
   /**
    * Remove FIELD columns that are entirely null within {@code [0, rowSize)} 
according to BitMap.
    * TAG / ATTRIBUTE columns are always kept. Does not mutate the input tablet.
diff --git 
a/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
 
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
new file mode 100644
index 00000000000..fb785f3d93d
--- /dev/null
+++ 
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.rpccompress;
+
+import org.apache.iotdb.session.util.SessionUtils;
+
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encoding.encoder.PlainEncoder;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.DateUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * The PLAIN path of {@link TabletEncoder} must write exactly the bytes of the 
reference layout: the
+ * generic {@link PlainEncoder} for the fixed-width types (including the 
"repeat the last non-null
+ * value" rule and the canonical NaN), fixed-length ints for INT32 and DATE, 
length-prefixed bytes
+ * for the binary types, followed by the bit maps. Every codec must decompress 
to these bytes.
+ */
+public class TabletEncoderPlainTest {
+
+  private static final TSDataType[] TYPES = {
+    TSDataType.INT64,
+    TSDataType.TIMESTAMP,
+    TSDataType.DOUBLE,
+    TSDataType.FLOAT,
+    TSDataType.BOOLEAN,
+    TSDataType.INT32,
+    TSDataType.DATE,
+    TSDataType.TEXT,
+    TSDataType.STRING,
+    TSDataType.BLOB
+  };
+
+  // NaNs whose bits differ from Double.NaN / Float.NaN; PlainEncoder writes 
the canonical bits
+  private static final double NON_CANONICAL_DOUBLE_NAN =
+      Double.longBitsToDouble(0x7ff8000000000001L);
+  private static final float NON_CANONICAL_FLOAT_NAN = 
Float.intBitsToFloat(0x7fc00001);
+
+  @Test
+  public void plainPathMatchesTheReferenceForEveryCodec() throws IOException {
+    for (CompressionType compressionType : CompressionType.values()) {
+      Random random = new Random(42);
+      for (int iteration = 0; iteration < 100; iteration++) {
+        // TabletEncoder sizes GZIP output too small for noisy data (a 
separate, existing issue),
+        // so GZIP gets data that compresses well
+        boolean compressible = compressionType == CompressionType.GZIP;
+        Tablet tablet = randomTablet(random, iteration, compressible);
+        TabletEncoder encoder =
+            new TabletEncoder(
+                compressionType, Collections.nCopies(TYPES.length + 1, 
TSEncoding.PLAIN));
+        String context = compressionType + ", iteration " + iteration;
+
+        Assert.assertArrayEquals(
+            "time column, " + context,
+            expectedTime(tablet),
+            uncompress(compressionType, encoder.encodeTime(tablet)));
+        Assert.assertArrayEquals(
+            "value columns and bit maps, " + context,
+            expectedValues(tablet),
+            uncompress(compressionType, encoder.encodeValues(tablet)));
+      }
+    }
+  }
+
+  /**
+   * A table-model tablet built with the raw constructor may have no value 
array for a TAG column
+   * that is entirely null. Encoding must not read it, whatever the encodings.
+   */
+  @Test
+  public void allNullBinaryColumnWithoutValueArray() {
+    int rows = 10;
+    List<IMeasurementSchema> schemas =
+        Arrays.asList(
+            new MeasurementSchema("tag", TSDataType.STRING),
+            new MeasurementSchema("value", TSDataType.INT64));
+    List<ColumnCategory> categories = Arrays.asList(ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    long[] timestamps = new long[rows];
+    long[] longs = new long[rows];
+    for (int row = 0; row < rows; row++) {
+      timestamps[row] = row;
+      longs[row] = row * 10L;
+    }
+    BitMap allNull = new BitMap(rows);
+    allNull.markAll();
+
+    Tablet withoutArray =
+        new Tablet(
+            "table1",
+            schemas,
+            categories,
+            timestamps,
+            new Object[] {null, longs},
+            new BitMap[] {allNull, null},
+            rows);
+    Tablet withArray =
+        new Tablet(
+            "table1",
+            schemas,
+            categories,
+            timestamps,
+            new Object[] {new Binary[rows], longs},
+            new BitMap[] {allNull, null},
+            rows);
+
+    for (TSEncoding valueEncoding : new TSEncoding[] {TSEncoding.PLAIN, 
TSEncoding.TS_2DIFF}) {
+      TabletEncoder encoder =
+          new TabletEncoder(
+              CompressionType.UNCOMPRESSED,
+              Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN, 
valueEncoding));
+      Assert.assertEquals(
+          "values, " + valueEncoding,
+          encoder.encodeValues(withArray),
+          encoder.encodeValues(withoutArray));
+    }
+  }
+
+  private static byte[] expectedTime(Tablet tablet) {
+    PublicBAOS expected = new PublicBAOS();
+    Encoder timeEncoder = new PlainEncoder(TSDataType.INT64, 0);
+    for (int row = 0; row < tablet.getRowSize(); row++) {
+      timeEncoder.encode(tablet.getTimestamp(row), expected);
+    }
+    return Arrays.copyOf(expected.getBuf(), expected.size());
+  }
+
+  private static byte[] expectedValues(Tablet tablet) throws IOException {
+    PublicBAOS expected = new PublicBAOS();
+    for (int column = 0; column < TYPES.length; column++) {
+      TSDataType type = TYPES[column];
+      Object values = tablet.getValues()[column];
+      for (int row = 0; row < tablet.getRowSize(); row++) {
+        boolean isNull = tablet.isNull(row, column);
+        switch (type) {
+          case INT32:
+            ReadWriteIOUtils.write(((int[]) values)[row], expected);
+            break;
+          case DATE:
+            ReadWriteIOUtils.write(
+                isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) 
values)[row]),
+                expected);
+            break;
+          case TEXT:
+          case STRING:
+          case BLOB:
+            ReadWriteIOUtils.write(
+                isNull ? Binary.EMPTY_VALUE : ((Binary[]) values)[row], 
expected);
+            break;
+          default:
+            // the fixed-width types are checked column-wise against 
PlainEncoder below
+            break;
+        }
+      }
+      if (!type.isBinary() && type != TSDataType.INT32 && type != 
TSDataType.DATE) {
+        SessionUtils.encodeValue(type, tablet, column, new PlainEncoder(type, 
0), expected);
+      }
+    }
+    for (BitMap bitMap : tablet.getBitMaps()) {
+      boolean columnHasNull = bitMap != null && 
!bitMap.isAllUnmarked(tablet.getRowSize());
+      expected.write(BytesUtils.boolToByte(columnHasNull));
+      if (columnHasNull) {
+        expected.write(bitMap.getTruncatedByteArray(tablet.getRowSize()));
+      }
+    }
+    return Arrays.copyOf(expected.getBuf(), expected.size());
+  }
+
+  private static Tablet randomTablet(Random random, int iteration, boolean 
compressible) {
+    List<IMeasurementSchema> schemas = new ArrayList<>();
+    for (int column = 0; column < TYPES.length; column++) {
+      schemas.add(new MeasurementSchema("s" + column, TYPES[column]));
+    }
+    int rows = 1 + random.nextInt(300);
+    Tablet tablet = new Tablet("root.sg.d1", schemas, rows);
+    for (int row = 0; row < rows; row++) {
+      tablet.addTimestamp(row, compressible ? row : random.nextLong());
+      for (int column = 0; column < TYPES.length; column++) {
+        if (random.nextInt(5) == 0) {
+          continue; // leave a null
+        }
+        int small = random.nextInt(4);
+        switch (TYPES[column]) {
+          case INT64:
+          case TIMESTAMP:
+            tablet.addValue(row, column, compressible ? (long) small : 
random.nextLong());
+            break;
+          case DOUBLE:
+            tablet.addValue(row, column, randomDouble(random, iteration, 
compressible, small));
+            break;
+          case FLOAT:
+            tablet.addValue(row, column, randomFloat(random, iteration, 
compressible, small));
+            break;
+          case BOOLEAN:
+            tablet.addValue(row, column, random.nextBoolean());
+            break;
+          case INT32:
+            tablet.addValue(row, column, compressible ? small : 
random.nextInt());
+            break;
+          case DATE:
+            tablet.addValue(row, column, LocalDate.of(2000 + small, 1 + 
random.nextInt(12), 1));
+            break;
+          case TEXT:
+          case STRING:
+          case BLOB:
+            tablet.addValue(
+                row,
+                column,
+                ("v" + (compressible ? small : 
random.nextInt())).getBytes(StandardCharsets.UTF_8));
+            break;
+          default:
+            throw new IllegalStateException();
+        }
+      }
+    }
+    return tablet;
+  }
+
+  private static double randomDouble(
+      Random random, int iteration, boolean compressible, int small) {
+    if (iteration % 7 == 0 && small == 0) {
+      return random.nextBoolean() ? Double.NaN : NON_CANONICAL_DOUBLE_NAN;
+    }
+    return compressible ? small : random.nextGaussian();
+  }
+
+  private static float randomFloat(Random random, int iteration, boolean 
compressible, int small) {
+    if (iteration % 7 == 0 && small == 0) {
+      return random.nextBoolean() ? Float.NaN : NON_CANONICAL_FLOAT_NAN;
+    }
+    return compressible ? small : random.nextFloat() - 0.5f;
+  }
+
+  /** Undoes TabletEncoder's compression: the compressed bytes, then the 
uncompressed size. */
+  private static byte[] uncompress(CompressionType compressionType, ByteBuffer 
buffer)
+      throws IOException {
+    byte[] bytes = new byte[buffer.remaining()];
+    buffer.get(bytes);
+    if (compressionType == CompressionType.UNCOMPRESSED) {
+      return bytes;
+    }
+    int uncompressedSize =
+        ByteBuffer.wrap(bytes, bytes.length - Integer.BYTES, 
Integer.BYTES).getInt();
+    byte[] uncompressed = new byte[uncompressedSize];
+    int length =
+        IUnCompressor.getUnCompressor(compressionType)
+            .uncompress(bytes, 0, bytes.length - Integer.BYTES, uncompressed, 
0);
+    Assert.assertEquals(compressionType + " uncompressed size", 
uncompressedSize, length);
+    return uncompressed;
+  }
+}

Reply via email to