This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c46fe4b8490 [Session] Write PLAIN-encoded tablet columns in bulk
instead of byte by byte (#19176)
c46fe4b8490 is described below
commit c46fe4b84906ca8720dd13b319ec9dbe716c2f71
Author: Christofer Dutz <[email protected]>
AuthorDate: Sun Nov 8 14:09:55 2026 +0100
[Session] Write PLAIN-encoded tablet columns in bulk instead of byte by
byte (#19176)
---
.../iotdb/session/rpccompress/TabletEncoder.java | 81 ++----
.../iotdb/session/util/SessionTypeServices.java | 141 ++++++++++
.../apache/iotdb/session/util/SessionUtils.java | 17 ++
.../rpccompress/TabletEncoderPlainTest.java | 289 +++++++++++++++++++++
4 files changed, 475 insertions(+), 53 deletions(-)
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
index be44c5feb20..53964ec0ae8 100644
---
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
+++
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/rpccompress/TabletEncoder.java
@@ -27,12 +27,9 @@ import org.apache.tsfile.encoding.encoder.TSEncodingBuilder;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.BytesUtils;
-import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.utils.PublicBAOS;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -40,7 +37,6 @@ import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.time.LocalDate;
import java.util.List;
public class TabletEncoder {
@@ -57,6 +53,10 @@ public class TabletEncoder {
}
public ByteBuffer encodeTime(Tablet tablet) {
+ if (encodingList.get(0) == TSEncoding.PLAIN) {
+ // the same bytes as PlainEncoder (big-endian longs), written in one go
+ return compressBuffer(SessionUtils.getTimeBuffer(tablet));
+ }
Encoder encoder =
TSEncodingBuilder.getEncodingBuilder(encodingList.get(0)).getEncoder(TSDataType.INT64);
PublicBAOS baos = new PublicBAOS();
@@ -74,25 +74,16 @@ public class TabletEncoder {
}
public ByteBuffer encodeValues(Tablet tablet) {
- PublicBAOS baos = new PublicBAOS();
List<IMeasurementSchema> schemas = tablet.getSchemas();
+ PublicBAOS baos = newValueStream(tablet, schemas);
for (int j = 0, schemasSize = schemas.size(); j < schemasSize; j++) {
- IMeasurementSchema schema = schemas.get(j);
- TSDataType dataType = schema.getType();
+ TSDataType dataType = schemas.get(j).getType();
TSEncoding encoding = encodingList.get(j + 1);
- Encoder encoder =
TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
if (encoding == TSEncoding.PLAIN) {
- // PlainEncoder uses var int, which may cause compatibility problem
- if (dataType.isBinary()) {
- encodePlainBinary(tablet, j, baos);
- } else if (dataType == TSDataType.INT32) {
- encodePlainI32(tablet, j, baos);
- } else if (dataType == TSDataType.DATE) {
- encodePlainDate(tablet, j, baos);
- } else {
- SessionUtils.encodeValue(dataType, tablet, j, encoder, baos);
- }
+ // not through PlainEncoder, which writes INT32 as a var int
+ SessionUtils.encodePlainValue(dataType, tablet, j, baos);
} else {
+ Encoder encoder =
TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
SessionUtils.encodeValue(dataType, tablet, j, encoder, baos);
}
}
@@ -103,45 +94,29 @@ public class TabletEncoder {
return compressBuffer(buffer);
}
- private void encodePlainBinary(Tablet tablet, int j, ByteArrayOutputStream
baos) {
- Binary[] binaryValues = (Binary[]) tablet.getValues()[j];
- try {
- for (int index = 0; index < tablet.getRowSize(); index++) {
- if (!tablet.isNull(index, j)) {
- ReadWriteIOUtils.write(binaryValues[index], baos);
- } else {
- ReadWriteIOUtils.write(Binary.EMPTY_VALUE, baos);
- }
+ /**
+ * Pre-sizes the stream for the PLAIN columns, whose size is known up front,
and the bit maps
+ * after them, so that the stream does not have to double for the last few
bytes. Other encodings
+ * and binary values let the stream grow, as does a size that does not fit
in an array.
+ */
+ private PublicBAOS newValueStream(Tablet tablet, List<IMeasurementSchema>
schemas) {
+ int rows = tablet.getRowSize();
+ long size = 0;
+ for (int j = 0, schemasSize = schemas.size(); j < schemasSize; j++) {
+ if (encodingList.get(j + 1) == TSEncoding.PLAIN) {
+ size += (long) rows *
SessionUtils.getPlainRowSize(schemas.get(j).getType());
}
- } catch (IOException e) {
- throw new IllegalStateException(e);
}
- }
-
- private void encodePlainI32(Tablet tablet, int j, ByteArrayOutputStream
baos) {
- int[] intValues = (int[]) tablet.getValues()[j];
- try {
- for (int index = 0; index < tablet.getRowSize(); index++) {
- ReadWriteIOUtils.write(intValues[index], baos);
- }
- } catch (IOException e) {
- throw new IllegalStateException(e);
+ if (size == 0) {
+ return new PublicBAOS();
}
- }
-
- private void encodePlainDate(Tablet tablet, int j, ByteArrayOutputStream
baos) {
- LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[j];
- try {
- for (int index = 0; index < tablet.getRowSize(); index++) {
- if (!tablet.isNull(index, j)) {
-
ReadWriteIOUtils.write(DateUtils.parseDateExpressionToInt(dateValues[index]),
baos);
- } else {
- ReadWriteIOUtils.write(0, baos);
- }
- }
- } catch (IOException e) {
- throw new IllegalStateException(e);
+ BitMap[] bitMaps = tablet.getBitMaps();
+ if (bitMaps != null) {
+ // at most a marker byte and the truncated bit map per column
+ size += bitMaps.length * (1L + rows / Byte.SIZE + 1);
}
+ // stay clear of the VM's array size limit
+ return size <= Integer.MAX_VALUE - 8 ? new PublicBAOS((int) size) : new
PublicBAOS();
}
private void encodeBitMap(Tablet tablet, ByteArrayOutputStream baos) {
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
index 0daca161a60..d4b26820fcb 100644
---
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
+++
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionTypeServices.java
@@ -35,6 +35,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.record.Tablet;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.LocalDate;
@@ -276,6 +277,127 @@ final class SessionTypeServices {
};
};
+ /**
+ * Writes a tablet column in the PLAIN layout that {@code TabletEncoder}
sends. The fixed-width
+ * types produce the same bytes as {@code PlainEncoder} through {@link
+ * #TABLET_VALUE_ENCODER_SERVICE} (big-endian; a null repeats the last
non-null value, 0 before
+ * the first one), but fill one buffer per column instead of making one
{@code write(int)} call,
+ * with its capacity check, per byte. INT32, DATE and the binary types keep
their fixed-length
+ * layout, because {@code PlainEncoder} writes INT32 as a var int.
+ */
+ private static final TypeService<TabletPlainValueWriter>
TABLET_PLAIN_VALUE_WRITER_SERVICE =
+ type ->
+ switch (type.getTypeEnum()) {
+ case INT32 ->
+ (tablet, columnIndex, outputStream) -> {
+ int[] values = (int[]) tablet.getValues()[columnIndex];
+ ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize()
* Integer.BYTES);
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ buffer.putInt(values[index]);
+ }
+ writeTo(buffer, outputStream);
+ };
+ case DATE ->
+ (tablet, columnIndex, outputStream) -> {
+ LocalDate[] values = (LocalDate[])
tablet.getValues()[columnIndex];
+ ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize()
* Integer.BYTES);
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ buffer.putInt(
+ tablet.isNull(index, columnIndex)
+ ? 0
+ :
DateUtils.parseDateExpressionToInt(values[index]));
+ }
+ writeTo(buffer, outputStream);
+ };
+ case INT64, TIMESTAMP ->
+ (tablet, columnIndex, outputStream) -> {
+ long[] values = (long[]) tablet.getValues()[columnIndex];
+ ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize()
* Long.BYTES);
+ long lastNonNullValue = 0;
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ if (!tablet.isNull(index, columnIndex)) {
+ lastNonNullValue = values[index];
+ }
+ buffer.putLong(lastNonNullValue);
+ }
+ writeTo(buffer, outputStream);
+ };
+ case FLOAT ->
+ (tablet, columnIndex, outputStream) -> {
+ float[] values = (float[]) tablet.getValues()[columnIndex];
+ ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize()
* Float.BYTES);
+ float lastNonNullValue = 0.0f;
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ if (!tablet.isNull(index, columnIndex)) {
+ lastNonNullValue = values[index];
+ }
+ // floatToIntBits like PlainEncoder, which collapses NaN
payloads
+ buffer.putInt(Float.floatToIntBits(lastNonNullValue));
+ }
+ writeTo(buffer, outputStream);
+ };
+ case DOUBLE ->
+ (tablet, columnIndex, outputStream) -> {
+ double[] values = (double[]) tablet.getValues()[columnIndex];
+ ByteBuffer buffer = ByteBuffer.allocate(tablet.getRowSize()
* Double.BYTES);
+ double lastNonNullValue = 0.0;
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ if (!tablet.isNull(index, columnIndex)) {
+ lastNonNullValue = values[index];
+ }
+ // doubleToLongBits like PlainEncoder, which collapses NaN
payloads
+ buffer.putLong(Double.doubleToLongBits(lastNonNullValue));
+ }
+ writeTo(buffer, outputStream);
+ };
+ case BOOLEAN ->
+ (tablet, columnIndex, outputStream) -> {
+ boolean[] values = (boolean[])
tablet.getValues()[columnIndex];
+ byte[] bytes = new byte[tablet.getRowSize()];
+ boolean lastNonNullValue = false;
+ for (int index = 0; index < bytes.length; index++) {
+ if (!tablet.isNull(index, columnIndex)) {
+ lastNonNullValue = values[index];
+ }
+ bytes[index] = BytesUtils.boolToByte(lastNonNullValue);
+ }
+ outputStream.write(bytes, 0, bytes.length);
+ };
+ case TEXT, STRING, BLOB, OBJECT ->
+ (tablet, columnIndex, outputStream) -> {
+ Binary[] values = (Binary[]) tablet.getValues()[columnIndex];
+ try {
+ for (int index = 0; index < tablet.getRowSize(); index++) {
+ ReadWriteIOUtils.write(
+ tablet.isNull(index, columnIndex) ?
Binary.EMPTY_VALUE : values[index],
+ outputStream);
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ };
+ case ROW, UNKNOWN, VECTOR ->
+ (tablet, columnIndex, outputStream) -> {
+ throw unsupportedTabletDataType(type.getTypeEnum());
+ };
+ };
+
+ /**
+ * Bytes per row of a column in the PLAIN layout of {@link
#TABLET_PLAIN_VALUE_WRITER_SERVICE}, to
+ * pre-size the output. Binary types count only their length prefix, so that
no values are read;
+ * the stream grows for their bytes.
+ */
+ private static final TypeService<Integer> TABLET_PLAIN_ROW_SIZE_SERVICE =
+ type ->
+ switch (type.getTypeEnum()) {
+ case BOOLEAN -> Byte.BYTES;
+ case INT32, FLOAT, DATE -> Integer.BYTES;
+ case INT64, DOUBLE, TIMESTAMP -> Long.BYTES;
+ case TEXT, STRING, BLOB, OBJECT -> Integer.BYTES;
+ // the writer refuses these types
+ case ROW, UNKNOWN, VECTOR -> 0;
+ };
+
private static final TypeService<ValueListSorter> VALUE_LIST_SORTER_SERVICE =
type ->
switch (type.getTypeEnum()) {
@@ -354,6 +476,8 @@ final class SessionTypeServices {
TABLET_COLUMN_OCCUPATION_CALCULATOR_SERVICE.check();
TABLET_VALUE_WRITER_SERVICE.check();
TABLET_VALUE_ENCODER_SERVICE.check();
+ TABLET_PLAIN_VALUE_WRITER_SERVICE.check();
+ TABLET_PLAIN_ROW_SIZE_SERVICE.check();
VALUE_LIST_SORTER_SERVICE.check();
}
@@ -379,6 +503,14 @@ final class SessionTypeServices {
return TABLET_VALUE_ENCODER_SERVICE.call(Type.fromTsDataType(dataType));
}
+ static TabletPlainValueWriter tabletPlainValueWriter(TSDataType dataType) {
+ return
TABLET_PLAIN_VALUE_WRITER_SERVICE.call(Type.fromTsDataType(dataType));
+ }
+
+ static int tabletPlainRowSize(TSDataType dataType) {
+ return TABLET_PLAIN_ROW_SIZE_SERVICE.call(Type.fromTsDataType(dataType));
+ }
+
static ValueListSorter valueListSorter(TSDataType dataType) {
return VALUE_LIST_SORTER_SERVICE.call(Type.fromTsDataType(dataType));
}
@@ -390,6 +522,10 @@ final class SessionTypeServices {
return ((String) value).getBytes(TSFileConfig.STRING_CHARSET);
}
+ private static void writeTo(ByteBuffer buffer, ByteArrayOutputStream
outputStream) {
+ outputStream.write(buffer.array(), 0, buffer.position());
+ }
+
private static IoTDBConnectionException unsupportedDataType(Object dataType)
{
return new IoTDBConnectionException(Session.MSG_UNSUPPORTED_DATA_TYPE +
dataType);
}
@@ -429,6 +565,11 @@ final class SessionTypeServices {
Tablet tablet, int columnIndex, Encoder encoder, ByteArrayOutputStream
outputStream);
}
+ @FunctionalInterface
+ interface TabletPlainValueWriter {
+ void write(Tablet tablet, int columnIndex, ByteArrayOutputStream
outputStream);
+ }
+
@FunctionalInterface
interface ValueListSorter {
Object sort(Object valueList, int[] index);
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
index 5203fffd915..2692f1c600a 100644
---
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
+++
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
@@ -193,6 +193,23 @@ public class SessionUtils {
}
}
+ /**
+ * Writes a tablet column in the PLAIN layout of the RPC tablet compression:
fixed-width values
+ * big-endian (INT32 not as a var int), binary values with an int length
prefix.
+ */
+ public static void encodePlainValue(
+ TSDataType dataType, Tablet tablet, int i, ByteArrayOutputStream
outputStream) {
+ SessionTypeServices.tabletPlainValueWriter(dataType).write(tablet, i,
outputStream);
+ }
+
+ /**
+ * @return the bytes per row of a column in the layout of {@link
#encodePlainValue}, without the
+ * bytes of binary values
+ */
+ public static int getPlainRowSize(TSDataType dataType) {
+ return SessionTypeServices.tabletPlainRowSize(dataType);
+ }
+
/**
* Remove FIELD columns that are entirely null within {@code [0, rowSize)}
according to BitMap.
* TAG / ATTRIBUTE columns are always kept. Does not mutate the input tablet.
diff --git
a/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
new file mode 100644
index 00000000000..fb785f3d93d
--- /dev/null
+++
b/iotdb-client/session/src/test/java/org/apache/iotdb/session/rpccompress/TabletEncoderPlainTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.rpccompress;
+
+import org.apache.iotdb.session.util.SessionUtils;
+
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.encoding.encoder.Encoder;
+import org.apache.tsfile.encoding.encoder.PlainEncoder;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.DateUtils;
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * The PLAIN path of {@link TabletEncoder} must write exactly the bytes of the
reference layout: the
+ * generic {@link PlainEncoder} for the fixed-width types (including the
"repeat the last non-null
+ * value" rule and the canonical NaN), fixed-length ints for INT32 and DATE,
length-prefixed bytes
+ * for the binary types, followed by the bit maps. Every codec must decompress
to these bytes.
+ */
+public class TabletEncoderPlainTest {
+
+ private static final TSDataType[] TYPES = {
+ TSDataType.INT64,
+ TSDataType.TIMESTAMP,
+ TSDataType.DOUBLE,
+ TSDataType.FLOAT,
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.DATE,
+ TSDataType.TEXT,
+ TSDataType.STRING,
+ TSDataType.BLOB
+ };
+
+ // NaNs whose bits differ from Double.NaN / Float.NaN; PlainEncoder writes
the canonical bits
+ private static final double NON_CANONICAL_DOUBLE_NAN =
+ Double.longBitsToDouble(0x7ff8000000000001L);
+ private static final float NON_CANONICAL_FLOAT_NAN =
Float.intBitsToFloat(0x7fc00001);
+
+ @Test
+ public void plainPathMatchesTheReferenceForEveryCodec() throws IOException {
+ for (CompressionType compressionType : CompressionType.values()) {
+ Random random = new Random(42);
+ for (int iteration = 0; iteration < 100; iteration++) {
+ // TabletEncoder sizes GZIP output too small for noisy data (a
separate, existing issue),
+ // so GZIP gets data that compresses well
+ boolean compressible = compressionType == CompressionType.GZIP;
+ Tablet tablet = randomTablet(random, iteration, compressible);
+ TabletEncoder encoder =
+ new TabletEncoder(
+ compressionType, Collections.nCopies(TYPES.length + 1,
TSEncoding.PLAIN));
+ String context = compressionType + ", iteration " + iteration;
+
+ Assert.assertArrayEquals(
+ "time column, " + context,
+ expectedTime(tablet),
+ uncompress(compressionType, encoder.encodeTime(tablet)));
+ Assert.assertArrayEquals(
+ "value columns and bit maps, " + context,
+ expectedValues(tablet),
+ uncompress(compressionType, encoder.encodeValues(tablet)));
+ }
+ }
+ }
+
+ /**
+ * A table-model tablet built with the raw constructor may have no value
array for a TAG column
+ * that is entirely null. Encoding must not read it, whatever the encodings.
+ */
+ @Test
+ public void allNullBinaryColumnWithoutValueArray() {
+ int rows = 10;
+ List<IMeasurementSchema> schemas =
+ Arrays.asList(
+ new MeasurementSchema("tag", TSDataType.STRING),
+ new MeasurementSchema("value", TSDataType.INT64));
+ List<ColumnCategory> categories = Arrays.asList(ColumnCategory.TAG,
ColumnCategory.FIELD);
+ long[] timestamps = new long[rows];
+ long[] longs = new long[rows];
+ for (int row = 0; row < rows; row++) {
+ timestamps[row] = row;
+ longs[row] = row * 10L;
+ }
+ BitMap allNull = new BitMap(rows);
+ allNull.markAll();
+
+ Tablet withoutArray =
+ new Tablet(
+ "table1",
+ schemas,
+ categories,
+ timestamps,
+ new Object[] {null, longs},
+ new BitMap[] {allNull, null},
+ rows);
+ Tablet withArray =
+ new Tablet(
+ "table1",
+ schemas,
+ categories,
+ timestamps,
+ new Object[] {new Binary[rows], longs},
+ new BitMap[] {allNull, null},
+ rows);
+
+ for (TSEncoding valueEncoding : new TSEncoding[] {TSEncoding.PLAIN,
TSEncoding.TS_2DIFF}) {
+ TabletEncoder encoder =
+ new TabletEncoder(
+ CompressionType.UNCOMPRESSED,
+ Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN,
valueEncoding));
+ Assert.assertEquals(
+ "values, " + valueEncoding,
+ encoder.encodeValues(withArray),
+ encoder.encodeValues(withoutArray));
+ }
+ }
+
+ private static byte[] expectedTime(Tablet tablet) {
+ PublicBAOS expected = new PublicBAOS();
+ Encoder timeEncoder = new PlainEncoder(TSDataType.INT64, 0);
+ for (int row = 0; row < tablet.getRowSize(); row++) {
+ timeEncoder.encode(tablet.getTimestamp(row), expected);
+ }
+ return Arrays.copyOf(expected.getBuf(), expected.size());
+ }
+
+ private static byte[] expectedValues(Tablet tablet) throws IOException {
+ PublicBAOS expected = new PublicBAOS();
+ for (int column = 0; column < TYPES.length; column++) {
+ TSDataType type = TYPES[column];
+ Object values = tablet.getValues()[column];
+ for (int row = 0; row < tablet.getRowSize(); row++) {
+ boolean isNull = tablet.isNull(row, column);
+ switch (type) {
+ case INT32:
+ ReadWriteIOUtils.write(((int[]) values)[row], expected);
+ break;
+ case DATE:
+ ReadWriteIOUtils.write(
+ isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[])
values)[row]),
+ expected);
+ break;
+ case TEXT:
+ case STRING:
+ case BLOB:
+ ReadWriteIOUtils.write(
+ isNull ? Binary.EMPTY_VALUE : ((Binary[]) values)[row],
expected);
+ break;
+ default:
+ // the fixed-width types are checked column-wise against
PlainEncoder below
+ break;
+ }
+ }
+ if (!type.isBinary() && type != TSDataType.INT32 && type !=
TSDataType.DATE) {
+ SessionUtils.encodeValue(type, tablet, column, new PlainEncoder(type,
0), expected);
+ }
+ }
+ for (BitMap bitMap : tablet.getBitMaps()) {
+ boolean columnHasNull = bitMap != null &&
!bitMap.isAllUnmarked(tablet.getRowSize());
+ expected.write(BytesUtils.boolToByte(columnHasNull));
+ if (columnHasNull) {
+ expected.write(bitMap.getTruncatedByteArray(tablet.getRowSize()));
+ }
+ }
+ return Arrays.copyOf(expected.getBuf(), expected.size());
+ }
+
+ private static Tablet randomTablet(Random random, int iteration, boolean
compressible) {
+ List<IMeasurementSchema> schemas = new ArrayList<>();
+ for (int column = 0; column < TYPES.length; column++) {
+ schemas.add(new MeasurementSchema("s" + column, TYPES[column]));
+ }
+ int rows = 1 + random.nextInt(300);
+ Tablet tablet = new Tablet("root.sg.d1", schemas, rows);
+ for (int row = 0; row < rows; row++) {
+ tablet.addTimestamp(row, compressible ? row : random.nextLong());
+ for (int column = 0; column < TYPES.length; column++) {
+ if (random.nextInt(5) == 0) {
+ continue; // leave a null
+ }
+ int small = random.nextInt(4);
+ switch (TYPES[column]) {
+ case INT64:
+ case TIMESTAMP:
+ tablet.addValue(row, column, compressible ? (long) small :
random.nextLong());
+ break;
+ case DOUBLE:
+ tablet.addValue(row, column, randomDouble(random, iteration,
compressible, small));
+ break;
+ case FLOAT:
+ tablet.addValue(row, column, randomFloat(random, iteration,
compressible, small));
+ break;
+ case BOOLEAN:
+ tablet.addValue(row, column, random.nextBoolean());
+ break;
+ case INT32:
+ tablet.addValue(row, column, compressible ? small :
random.nextInt());
+ break;
+ case DATE:
+ tablet.addValue(row, column, LocalDate.of(2000 + small, 1 +
random.nextInt(12), 1));
+ break;
+ case TEXT:
+ case STRING:
+ case BLOB:
+ tablet.addValue(
+ row,
+ column,
+ ("v" + (compressible ? small :
random.nextInt())).getBytes(StandardCharsets.UTF_8));
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+ }
+ return tablet;
+ }
+
+ private static double randomDouble(
+ Random random, int iteration, boolean compressible, int small) {
+ if (iteration % 7 == 0 && small == 0) {
+ return random.nextBoolean() ? Double.NaN : NON_CANONICAL_DOUBLE_NAN;
+ }
+ return compressible ? small : random.nextGaussian();
+ }
+
+ private static float randomFloat(Random random, int iteration, boolean
compressible, int small) {
+ if (iteration % 7 == 0 && small == 0) {
+ return random.nextBoolean() ? Float.NaN : NON_CANONICAL_FLOAT_NAN;
+ }
+ return compressible ? small : random.nextFloat() - 0.5f;
+ }
+
+ /** Undoes TabletEncoder's compression: the compressed bytes, then the
uncompressed size. */
+ private static byte[] uncompress(CompressionType compressionType, ByteBuffer
buffer)
+ throws IOException {
+ byte[] bytes = new byte[buffer.remaining()];
+ buffer.get(bytes);
+ if (compressionType == CompressionType.UNCOMPRESSED) {
+ return bytes;
+ }
+ int uncompressedSize =
+ ByteBuffer.wrap(bytes, bytes.length - Integer.BYTES,
Integer.BYTES).getInt();
+ byte[] uncompressed = new byte[uncompressedSize];
+ int length =
+ IUnCompressor.getUnCompressor(compressionType)
+ .uncompress(bytes, 0, bytes.length - Integer.BYTES, uncompressed,
0);
+ Assert.assertEquals(compressionType + " uncompressed size",
uncompressedSize, length);
+ return uncompressed;
+ }
+}