This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d7cf590774 [IOTDB-3384] fix run length tsblock serde (#6270)
d7cf590774 is described below
commit d7cf5907744dd2ccde99299bd25c657fc973fd1b
Author: Zhong Wang <[email protected]>
AuthorDate: Tue Jun 14 15:59:00 2022 +0800
[IOTDB-3384] fix run length tsblock serde (#6270)
---
.../block/column/BinaryArrayColumnEncoder.java | 5 +-
.../block/column/ByteArrayColumnEncoder.java | 5 +-
.../read/common/block/column/ColumnEncoder.java | 9 ++-
.../common/block/column/ColumnEncoderFactory.java | 3 +-
.../read/common/block/column/ColumnEncoding.java | 6 +-
.../block/column/Int32ArrayColumnEncoder.java | 8 ++-
.../block/column/Int64ArrayColumnEncoder.java | 21 +++++-
.../block/column/RunLengthColumnEncoder.java | 54 ++++++++++++++++
.../block/column/RunLengthEncodedColumn.java | 2 +-
.../read/common/block/column/TsBlockSerde.java | 19 +++---
.../common/block/BinaryArrayColumnEncoderTest.java | 8 +--
.../common/block/ByteArrayColumnEncoderTest.java | 9 +--
.../common/block/Int32ArrayColumnEncoderTest.java | 15 ++---
.../common/block/Int64ArrayColumnEncoderTest.java | 17 ++---
...erTest.java => RunLengthColumnEncoderTest.java} | 75 ++++++++++++++--------
15 files changed, 181 insertions(+), 75 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
index 9feeaa0197..715bfe7786 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
@@ -29,7 +29,7 @@ import java.nio.ByteBuffer;
public class BinaryArrayColumnEncoder implements ColumnEncoder {
@Override
- public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int
positionCount) {
+ public Column readColumn(ByteBuffer input, TSDataType dataType, int
positionCount) {
// Serialized data layout:
// +---------------+-----------------+-------------+
// | may have null | null indicators | values |
@@ -46,8 +46,8 @@ public class BinaryArrayColumnEncoder implements
ColumnEncoder {
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input,
positionCount);
- TSDataType dataType = columnBuilder.getDataType();
if (TSDataType.TEXT.equals(dataType)) {
+ ColumnBuilder columnBuilder = new BinaryColumnBuilder(null,
positionCount);
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
int length = input.getInt();
@@ -58,6 +58,7 @@ public class BinaryArrayColumnEncoder implements
ColumnEncoder {
columnBuilder.appendNull();
}
}
+ return columnBuilder.build();
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
index 75bf1df701..f4fac3a1f0 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
@@ -28,7 +28,7 @@ import java.nio.ByteBuffer;
public class ByteArrayColumnEncoder implements ColumnEncoder {
@Override
- public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int
positionCount) {
+ public Column readColumn(ByteBuffer input, TSDataType dataType, int
positionCount) {
// Serialized data layout:
// +---------------+-----------------+-------------+
@@ -38,8 +38,8 @@ public class ByteArrayColumnEncoder implements ColumnEncoder {
// +---------------+-----------------+-------------+
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input,
positionCount);
- TSDataType dataType = columnBuilder.getDataType();
if (TSDataType.BOOLEAN.equals(dataType)) {
+ BooleanColumnBuilder columnBuilder = new BooleanColumnBuilder(null,
positionCount);
boolean[] values = ColumnEncoder.deserializeBooleanArray(input,
positionCount);
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
@@ -48,6 +48,7 @@ public class ByteArrayColumnEncoder implements ColumnEncoder {
columnBuilder.appendNull();
}
}
+ return columnBuilder.build();
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
index e24639549c..7bf79c11e6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
@@ -19,14 +19,21 @@
package org.apache.iotdb.tsfile.read.common.block.column;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
public interface ColumnEncoder {
+ /** Read a time column from the specified input. */
+ default TimeColumn readTimeColumn(ByteBuffer input, int positionCount) {
+ throw new UnsupportedOperationException();
+ }
+
/** Read a column from the specified input. */
- void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int
positionCount);
+ Column readColumn(ByteBuffer input, TSDataType dataType, int positionCount);
/** Write the specified column to the specified output */
void writeColumn(DataOutputStream output, Column column) throws IOException;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
index 4c08a17686..18c93afeea 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
@@ -24,13 +24,14 @@ import java.util.Map;
public class ColumnEncoderFactory {
- private static Map<ColumnEncoding, ColumnEncoder> encodingToEncoder = new
HashMap<>();
+ private static final Map<ColumnEncoding, ColumnEncoder> encodingToEncoder =
new HashMap<>();
static {
encodingToEncoder.put(ColumnEncoding.INT32_ARRAY, new
Int32ArrayColumnEncoder());
encodingToEncoder.put(ColumnEncoding.INT64_ARRAY, new
Int64ArrayColumnEncoder());
encodingToEncoder.put(ColumnEncoding.BYTE_ARRAY, new
ByteArrayColumnEncoder());
encodingToEncoder.put(ColumnEncoding.BINARY_ARRAY, new
BinaryArrayColumnEncoder());
+ encodingToEncoder.put(ColumnEncoding.RLE, new RunLengthColumnEncoder());
}
public static ColumnEncoder get(ColumnEncoding columnEncoding) {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoding.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoding.java
index 0c7f2a70b8..f8008ae288 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoding.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoding.java
@@ -31,7 +31,9 @@ public enum ColumnEncoding {
/** INT64, DOUBLE. */
INT64_ARRAY((byte) 2),
/** TEXT. */
- BINARY_ARRAY((byte) 3);
+ BINARY_ARRAY((byte) 3),
+ /** All data types. */
+ RLE((byte) 4);
private final byte value;
@@ -57,6 +59,8 @@ public enum ColumnEncoding {
return INT64_ARRAY;
case 3:
return BINARY_ARRAY;
+ case 4:
+ return RLE;
default:
throw new IllegalArgumentException("Invalid value: " + value);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
index da95bd8cae..23339473fb 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
@@ -28,7 +28,7 @@ import java.nio.ByteBuffer;
public class Int32ArrayColumnEncoder implements ColumnEncoder {
@Override
- public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int
positionCount) {
+ public Column readColumn(ByteBuffer input, TSDataType dataType, int
positionCount) {
// Serialized data layout:
// +---------------+-----------------+-------------+
@@ -39,8 +39,9 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder
{
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input,
positionCount);
- TSDataType dataType = columnBuilder.getDataType();
+ ColumnBuilder columnBuilder;
if (TSDataType.INT32.equals(dataType)) {
+ columnBuilder = new IntColumnBuilder(null, positionCount);
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeInt(input.getInt());
@@ -49,6 +50,7 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder
{
}
}
} else if (TSDataType.FLOAT.equals(dataType)) {
+ columnBuilder = new FloatColumnBuilder(null, positionCount);
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeFloat(Float.intBitsToFloat(input.getInt()));
@@ -59,6 +61,8 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder
{
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
}
+
+ return columnBuilder.build();
}
@Override
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
index a7f1154d4a..9264564185 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
@@ -28,7 +28,24 @@ import java.nio.ByteBuffer;
public class Int64ArrayColumnEncoder implements ColumnEncoder {
@Override
- public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int
positionCount) {
+ public TimeColumn readTimeColumn(ByteBuffer input, int positionCount) {
+ return (TimeColumn)
+ readColumnInternal(new TimeColumnBuilder(null, positionCount), input,
positionCount);
+ }
+
+ @Override
+ public Column readColumn(ByteBuffer input, TSDataType dataType, int
positionCount) {
+ if (TSDataType.INT64.equals(dataType)) {
+ return readColumnInternal(new LongColumnBuilder(null, positionCount),
input, positionCount);
+ } else if (TSDataType.DOUBLE.equals(dataType)) {
+ return readColumnInternal(new DoubleColumnBuilder(null, positionCount),
input, positionCount);
+ } else {
+ throw new IllegalArgumentException("Invalid data type: " + dataType);
+ }
+ }
+
+ private Column readColumnInternal(
+ ColumnBuilder columnBuilder, ByteBuffer input, int positionCount) {
// Serialized data layout:
// +---------------+-----------------+-------------+
@@ -38,7 +55,6 @@ public class Int64ArrayColumnEncoder implements ColumnEncoder
{
// +---------------+-----------------+-------------+
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input,
positionCount);
-
TSDataType dataType = columnBuilder.getDataType();
if (TSDataType.INT64.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
@@ -59,6 +75,7 @@ public class Int64ArrayColumnEncoder implements ColumnEncoder
{
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
}
+ return columnBuilder.build();
}
@Override
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthColumnEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthColumnEncoder.java
new file mode 100644
index 0000000000..9ee0665e31
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthColumnEncoder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read.common.block.column;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class RunLengthColumnEncoder implements ColumnEncoder {
+
+ @Override
+ public Column readColumn(ByteBuffer input, TSDataType dataType, int
positionCount) {
+ // Serialized data layout:
+ // +-----------+-------------------------+
+ // | encoding | serialized inner column |
+ // +-----------+-------------------------+
+ // | byte | list[byte] |
+ // +-----------+-------------------------+
+ ColumnEncoder columnEncoder =
ColumnEncoderFactory.get(ColumnEncoding.deserializeFrom(input));
+ Column innerColumn = columnEncoder.readColumn(input, dataType, 1);
+ return new RunLengthEncodedColumn(innerColumn, positionCount);
+ }
+
+ @Override
+ public void writeColumn(DataOutputStream output, Column column) throws
IOException {
+ Column innerColumn = ((RunLengthEncodedColumn) column).getValue();
+ if (innerColumn instanceof RunLengthEncodedColumn) {
+ throw new IOException("Unable to encode a nested RLE column.");
+ }
+
+ innerColumn.getEncoding().serializeTo(output);
+ ColumnEncoder columnEncoder =
ColumnEncoderFactory.get(innerColumn.getEncoding());
+ columnEncoder.writeColumn(output, innerColumn);
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java
index b33995ee92..2dc69d43ec 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java
@@ -69,7 +69,7 @@ public class RunLengthEncodedColumn implements Column {
@Override
public ColumnEncoding getEncoding() {
- return value.getEncoding();
+ return ColumnEncoding.RLE;
}
@Override
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TsBlockSerde.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TsBlockSerde.java
index ca1b9f22c0..16eaf04003 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TsBlockSerde.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TsBlockSerde.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.tsfile.read.common.block.column;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
@@ -59,9 +58,6 @@ public class TsBlockSerde {
// Position count.
int positionCount = byteBuffer.getInt();
- TsBlockBuilder builder = new TsBlockBuilder(positionCount,
valueColumnDataTypes);
- builder.declarePositions(positionCount);
-
// Column encodings.
List<ColumnEncoding> columnEncodings = new ArrayList<>(valueColumnCount +
1);
for (int i = 0; i < valueColumnCount + 1; i++) {
@@ -69,18 +65,19 @@ public class TsBlockSerde {
}
// Time column.
- TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder();
- ColumnEncoderFactory.get(columnEncodings.get(0))
- .readColumn(timeColumnBuilder, byteBuffer, positionCount);
+ TimeColumn timeColumn =
+
ColumnEncoderFactory.get(columnEncodings.get(0)).readTimeColumn(byteBuffer,
positionCount);
+ // Value columns
+ Column[] valueColumns = new Column[valueColumnCount];
for (int i = 0; i < valueColumnCount; i++) {
// Value column.
- ColumnBuilder columnBuilder = builder.getColumnBuilder(i);
- ColumnEncoderFactory.get(columnEncodings.get(1 + i))
- .readColumn(columnBuilder, byteBuffer, positionCount);
+ valueColumns[i] =
+ ColumnEncoderFactory.get(columnEncodings.get(1 + i))
+ .readColumn(byteBuffer, valueColumnDataTypes.get(i),
positionCount);
}
- return builder.build();
+ return new TsBlock(positionCount, timeColumn, valueColumns);
}
/**
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
index 4a63caa9bf..c3d08d02e8 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
@@ -19,8 +19,8 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
@@ -55,6 +55,7 @@ public class BinaryArrayColumnEncoderTest {
}
}
BinaryColumn input = new BinaryColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.BINARY_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -67,11 +68,10 @@ public class BinaryArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null,
positionCount);
- encoder.readColumn(binaryColumnBuilder, buffer, positionCount);
- BinaryColumn output = (BinaryColumn) binaryColumnBuilder.build();
+ BinaryColumn output = (BinaryColumn) encoder.readColumn(buffer,
TSDataType.TEXT, positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
index b4457445a4..28bde621e7 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
@@ -19,8 +19,8 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.column.BooleanColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.BooleanColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
@@ -51,6 +51,7 @@ public class ByteArrayColumnEncoderTest {
}
}
BooleanColumn input = new BooleanColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.BYTE_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -63,11 +64,11 @@ public class ByteArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- BooleanColumnBuilder booleanColumnBuilder = new BooleanColumnBuilder(null,
positionCount);
- encoder.readColumn(booleanColumnBuilder, buffer, positionCount);
- BooleanColumn output = (BooleanColumn) booleanColumnBuilder.build();
+ BooleanColumn output =
+ (BooleanColumn) encoder.readColumn(buffer, TSDataType.BOOLEAN,
positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
index 3b3dca7fb9..09d9ab1977 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
@@ -19,13 +19,12 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
import org.apache.iotdb.tsfile.read.common.block.column.FloatColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.FloatColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.IntColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.IntColumnBuilder;
import org.junit.Assert;
import org.junit.Test;
@@ -53,6 +52,7 @@ public class Int32ArrayColumnEncoderTest {
}
}
IntColumn input = new IntColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -65,11 +65,10 @@ public class Int32ArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- IntColumnBuilder intColumnBuilder = new IntColumnBuilder(null,
positionCount);
- encoder.readColumn(intColumnBuilder, buffer, positionCount);
- IntColumn output = (IntColumn) intColumnBuilder.build();
+ IntColumn output = (IntColumn) encoder.readColumn(buffer,
TSDataType.INT32, positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
@@ -93,6 +92,7 @@ public class Int32ArrayColumnEncoderTest {
}
}
FloatColumn input = new FloatColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -105,11 +105,10 @@ public class Int32ArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- FloatColumnBuilder floatColumnBuilder = new FloatColumnBuilder(null,
positionCount);
- encoder.readColumn(floatColumnBuilder, buffer, positionCount);
- FloatColumn output = (FloatColumn) floatColumnBuilder.build();
+ FloatColumn output = (FloatColumn) encoder.readColumn(buffer,
TSDataType.FLOAT, positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
index 048801f1da..31e140535f 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
@@ -19,13 +19,12 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.LongColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.LongColumnBuilder;
import org.junit.Assert;
import org.junit.Test;
@@ -54,6 +53,7 @@ public class Int64ArrayColumnEncoderTest {
}
}
LongColumn input = new LongColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.INT64_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -66,11 +66,10 @@ public class Int64ArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- LongColumnBuilder longColumnBuilder = new LongColumnBuilder(null,
positionCount);
- encoder.readColumn(longColumnBuilder, buffer, positionCount);
- LongColumn output = (LongColumn) longColumnBuilder.build();
+ LongColumn output = (LongColumn) encoder.readColumn(buffer,
TSDataType.INT64, positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
@@ -94,6 +93,7 @@ public class Int64ArrayColumnEncoderTest {
}
}
DoubleColumn input = new DoubleColumn(positionCount,
Optional.of(nullIndicators), values);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.INT64_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -106,11 +106,12 @@ public class Int64ArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- DoubleColumnBuilder doubleColumnBuilder = new DoubleColumnBuilder(null,
positionCount);
- encoder.readColumn(doubleColumnBuilder, buffer, positionCount);
- DoubleColumn output = (DoubleColumn) doubleColumnBuilder.build();
+
+ DoubleColumn output =
+ (DoubleColumn) encoder.readColumn(buffer, TSDataType.DOUBLE,
positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/RunLengthColumnEncoderTest.java
similarity index 53%
copy from
tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
copy to
tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/RunLengthColumnEncoderTest.java
index 4a63caa9bf..07d926939d 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/RunLengthColumnEncoderTest.java
@@ -20,10 +20,16 @@
package org.apache.iotdb.tsfile.common.block;
import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.BooleanColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.Column;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
+import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.FloatColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.IntColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.LongColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.RunLengthEncodedColumn;
import org.apache.iotdb.tsfile.utils.Binary;
import org.junit.Assert;
@@ -34,28 +40,15 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Optional;
-import java.util.Random;
-public class BinaryArrayColumnEncoderTest {
- @Test
- public void testBinaryColumn() {
- final int positionCount = 10;
+public class RunLengthColumnEncoderTest {
- Random random = new Random();
+ private void testInternal(Column column) {
+ final int positionCount = 10;
- boolean[] nullIndicators = new boolean[positionCount];
- Binary[] values = new Binary[positionCount];
- for (int i = 0; i < positionCount; i++) {
- nullIndicators[i] = i % 2 == 0;
- if (i % 2 != 0) {
- int length = random.nextInt(1024) + 1;
- byte[] value = new byte[length];
- random.nextBytes(value);
- values[i] = new Binary(value);
- }
- }
- BinaryColumn input = new BinaryColumn(positionCount,
Optional.of(nullIndicators), values);
- ColumnEncoder encoder =
ColumnEncoderFactory.get(ColumnEncoding.BINARY_ARRAY);
+ Column input = new RunLengthEncodedColumn(column, positionCount);
+ long expectedRetainedSize = input.getRetainedSizeInBytes();
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.RLE);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
@@ -67,16 +60,42 @@ public class BinaryArrayColumnEncoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
- BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null,
positionCount);
- encoder.readColumn(binaryColumnBuilder, buffer, positionCount);
- BinaryColumn output = (BinaryColumn) binaryColumnBuilder.build();
+ Column output = encoder.readColumn(buffer, input.getDataType(),
positionCount);
Assert.assertEquals(positionCount, output.getPositionCount());
- Assert.assertTrue(output.mayHaveNull());
+ Assert.assertFalse(output.mayHaveNull());
+ Assert.assertEquals(expectedRetainedSize, output.getRetainedSizeInBytes());
for (int i = 0; i < positionCount; i++) {
- Assert.assertEquals(i % 2 == 0, output.isNull(i));
- if (i % 2 != 0) {
- Assert.assertEquals(values[i], output.getBinary(i));
- }
+ Assert.assertEquals(column.getObject(0), output.getObject(i));
}
}
+
+ @Test
+ public void testBooleanColumn() {
+ testInternal(new BooleanColumn(1, Optional.empty(), new boolean[] {true}));
+ }
+
+ @Test
+ public void testIntColumn() {
+ testInternal(new IntColumn(1, Optional.empty(), new int[] {0}));
+ }
+
+ @Test
+ public void testLongColumn() {
+ testInternal(new LongColumn(1, Optional.empty(), new long[] {0L}));
+ }
+
+ @Test
+ public void testFloatColumn() {
+ testInternal(new FloatColumn(1, Optional.empty(), new float[] {0.0F}));
+ }
+
+ @Test
+ public void testDoubleColumn() {
+ testInternal(new DoubleColumn(1, Optional.empty(), new double[] {0.0D}));
+ }
+
+ @Test
+ public void testTextColumn() {
+ testInternal(new BinaryColumn(1, Optional.empty(), new Binary[] {new
Binary("foo")}));
+ }
}
