This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch memtsblock in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 33acca88015eb9c4bba3010e0cdee38f7fdc45a6 Author: HTHou <[email protected]> AuthorDate: Mon May 2 21:44:01 2022 +0800 non-aligned --- .../db/engine/querycontext/ReadOnlyMemChunk.java | 59 +-------- .../db/metadata/utils/ResourceByPathUtils.java | 1 - .../db/utils/datastructure/AlignedTVList.java | 15 ++- .../iotdb/db/utils/datastructure/BinaryTVList.java | 33 +++-- .../db/utils/datastructure/BooleanTVList.java | 33 +++-- .../iotdb/db/utils/datastructure/DoubleTVList.java | 53 ++++---- .../iotdb/db/utils/datastructure/FloatTVList.java | 52 ++++---- .../iotdb/db/utils/datastructure/IntTVList.java | 33 +++-- .../iotdb/db/utils/datastructure/LongTVList.java | 33 +++-- .../iotdb/db/utils/datastructure/TVList.java | 144 ++++++++------------- .../db/engine/memtable/PrimitiveMemTableTest.java | 18 --- 11 files changed, 207 insertions(+), 267 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java index 40ce93b523..512b4565ec 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java @@ -78,7 +78,6 @@ public class ReadOnlyMemChunk { TSEncoding encoding, TVList tvList, Map<String, String> props, - int size, List<TimeRange> deletionList) throws IOException, QueryProcessException { this.measurementUid = measurementUid; @@ -102,56 +101,11 @@ public class ReadOnlyMemChunk { } this.chunkData = tvList; - this.chunkDataSize = size; + this.chunkDataSize = tvList.rowCount(); this.deletionList = deletionList; - - if (IoTDBDescriptor.getInstance().getConfig().isMppMode()) { - this.tsblock = tvList.getTsBlock(floatPrecision, encoding, chunkDataSize, deletionList); - initChunkMetaFromTsBlock(); - this.chunkPointReader = tsblock.getTsBlockSingleColumnIterator(); - } else { - this.chunkPointReader = - tvList.getIterator(floatPrecision, encoding, chunkDataSize, deletionList); - initChunkMeta(); - } - } - - private void initChunkMeta() throws IOException, QueryProcessException { - Statistics statsByType = Statistics.getStatsByType(dataType); - IChunkMetadata metaData = new ChunkMetadata(measurementUid, dataType, 0, statsByType); - if (!isEmpty()) { - IPointReader iterator = - chunkData.getIterator(floatPrecision, encoding, chunkDataSize, deletionList); - while (iterator.hasNextTimeValuePair()) { - TimeValuePair timeValuePair = iterator.nextTimeValuePair(); - switch (dataType) { - case BOOLEAN: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean()); - break; - case TEXT: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary()); - break; - case FLOAT: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat()); - break; - case INT32: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt()); - break; - case INT64: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong()); - break; - case DOUBLE: - statsByType.update(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble()); - break; - default: - throw new QueryProcessException("Unsupported data type:" + dataType); - } - } - } - statsByType.setEmpty(isEmpty()); - metaData.setChunkLoader(new MemChunkLoader(this)); - metaData.setVersion(Long.MAX_VALUE); - cachedMetaData = metaData; + this.tsblock = tvList.getTsBlock(floatPrecision, encoding, deletionList); + initChunkMetaFromTsBlock(); + this.chunkPointReader = tsblock.getTsBlockSingleColumnIterator(); } private void initChunkMetaFromTsBlock() throws IOException, QueryProcessException { @@ -159,7 +113,9 @@ public class ReadOnlyMemChunk { IChunkMetadata metaData = new ChunkMetadata(measurementUid, dataType, 0, statsByType); if (!isEmpty()) { IPointReader iterator = - chunkData.getTsBlock(floatPrecision, encoding, chunkDataSize, deletionList).getTsBlockSingleColumnIterator(); + chunkData + .getTsBlock(floatPrecision, encoding, deletionList) + .getTsBlockSingleColumnIterator(); while (iterator.hasNextTimeValuePair()) { TimeValuePair timeValuePair = iterator.nextTimeValuePair(); switch (dataType) { @@ -209,7 +165,6 @@ public class ReadOnlyMemChunk { } public IPointReader getPointReader() { - chunkPointReader = chunkData.getIterator(floatPrecision, encoding, chunkDataSize, deletionList); return chunkPointReader; } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java index c509b7d25e..9e4422ea53 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java @@ -550,7 +550,6 @@ class MeasurementResourceByPathUtils extends ResourceByPathUtils { partialPath.getMeasurementSchema().getEncodingType(), chunkCopy, partialPath.getMeasurementSchema().getProps(), - curSize, deletionList); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 4819d1cd89..03c846c59f 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -28,7 +28,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; +import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; import org.apache.iotdb.tsfile.read.reader.IPointReader; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.BitMap; @@ -917,10 +917,15 @@ public class AlignedTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - return null; - } + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) {} + + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) {} public IPointReader getAlignedIterator( int floatPrecision, diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java index fc15313619..fa78d82255 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.wal.utils.WALWriteUtils; @@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -203,19 +199,28 @@ public class BinaryTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); valueBuilder.writeBinaries(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - valueBuilder.writeBinaries(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + valueBuilder.writeBinaries( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); + } + + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeBoolean(getBoolean(i)); + } + } } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java index 93df9b173f..c2a648557a 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.wal.utils.WALWriteUtils; @@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -203,19 +199,28 @@ public class BooleanTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); valueBuilder.writeBooleans(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - valueBuilder.writeBooleans(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + valueBuilder.writeBooleans( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); + } + + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeBoolean(getBoolean(i)); + } + } } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java index 9abb886a70..39a64e1265 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; @@ -27,10 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -206,33 +202,46 @@ public class DoubleTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); - getValuesWithGivenPrecision(values.get(i), floatPrecision, encoding); + for (int j = 0; j < values.get(i).length; j++) { + values.get(i)[j] = roundValueWithGivenPrecision(values.get(i)[j], floatPrecision, encoding); + } valueBuilder.writeDoubles(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - getValuesWithGivenPrecision(values.get(values.size() - 1), floatPrecision, encoding); - valueBuilder.writeDoubles(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + for (int j = 0; j < values.get(values.size() - 1).length; j++) { + values.get(values.size() - 1)[j] = + roundValueWithGivenPrecision(values.get(values.size() - 1)[j], floatPrecision, encoding); + } + valueBuilder.writeDoubles( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); } - private void getValuesWithGivenPrecision(double[] values, int floatPrecision, TSEncoding encoding) { - if (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF) { - for (int i = 0; i < values.length; i++) { - if (!Double.isNaN(values[i])) { - values[i] = MathUtils.roundWithGivenPrecision(values[i], floatPrecision); - } + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeDouble( + roundValueWithGivenPrecision(getDouble(i), floatPrecision, encoding)); } } } + private double roundValueWithGivenPrecision( + double value, int floatPrecision, TSEncoding encoding) { + if (!Double.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + return MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + return value; + } + @Override protected void releaseLastValueArray() { PrimitiveArrayManager.release(values.remove(values.size() - 1)); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index 31eb193699..f010489059 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; @@ -27,10 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -206,33 +202,45 @@ public class FloatTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); - getValuesWithGivenPrecision(values.get(i), floatPrecision, encoding); + for (int j = 0; j < values.get(i).length; j++) { + values.get(i)[j] = roundValueWithGivenPrecision(values.get(i)[j], floatPrecision, encoding); + } valueBuilder.writeFloats(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - getValuesWithGivenPrecision(values.get(values.size() - 1), floatPrecision, encoding); - valueBuilder.writeFloats(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + for (int j = 0; j < values.get(values.size() - 1).length; j++) { + values.get(values.size() - 1)[j] = + roundValueWithGivenPrecision(values.get(values.size() - 1)[j], floatPrecision, encoding); + } + valueBuilder.writeFloats( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); } - private void getValuesWithGivenPrecision(float[] values, int floatPrecision, TSEncoding encoding) { - if (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF) { - for (int i = 0; i < values.length; i++) { - if (!Float.isNaN(values[i])) { - values[i] = MathUtils.roundWithGivenPrecision(values[i], floatPrecision); - } + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeFloat( + roundValueWithGivenPrecision(getFloat(i), floatPrecision, encoding)); } } } + private float roundValueWithGivenPrecision(float value, int floatPrecision, TSEncoding encoding) { + if (!Float.isNaN(value) && (encoding == TSEncoding.RLE || encoding == TSEncoding.TS_2DIFF)) { + return MathUtils.roundWithGivenPrecision(value, floatPrecision); + } + return value; + } + @Override protected void releaseLastValueArray() { PrimitiveArrayManager.release(values.remove(values.size() - 1)); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index e785c3ab91..e07d0c3876 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.wal.utils.WALWriteUtils; @@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -201,19 +197,28 @@ public class IntTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); valueBuilder.writeInts(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - valueBuilder.writeInts(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + valueBuilder.writeInts( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); + } + + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeInt(getInt(i)); + } + } } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java index 9d7f6d4af5..65d706a725 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.wal.utils.WALWriteUtils; @@ -26,10 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.common.block.TsBlock; -import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; -import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -201,19 +197,28 @@ public class LongTVList extends TVList { } @Override - public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, int size, - List<TimeRange> deletionList) { - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + protected void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding) { for (int i = 0; i < timestamps.size() - 1; i++) { - timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); valueBuilder.writeLongs(values.get(i), ARRAY_SIZE); } - timeBuilder.writeLongs(timestamps.get(timestamps.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - valueBuilder.writeLongs(values.get(values.size() - 1), size % ARRAY_SIZE == 0 ? ARRAY_SIZE : size % ARRAY_SIZE); - builder.declarePositions(size); - return builder.build(); + valueBuilder.writeLongs( + values.get(values.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); + } + + @Override + protected void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList) { + Integer deleteCursor = 0; + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + valueBuilder.writeLong(getLong(i)); + } + } } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index fb9f790bff..e9f13ef4fe 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.utils.datastructure; -import java.util.Collections; -import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.wal.buffer.WALEntryValue; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -31,7 +29,6 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlock; import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; -import org.apache.iotdb.tsfile.read.reader.IPointReader; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -39,6 +36,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -526,104 +524,68 @@ public abstract class TVList implements WALEntryValue { throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); } - @TestOnly - public IPointReader getIterator() { - return new Ite(); - } - - public IPointReader getIterator( - int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList) { - return new Ite(floatPrecision, encoding, size, deletionList); - } - - protected class Ite implements IPointReader { - - protected TimeValuePair cachedTimeValuePair; - protected boolean hasCachedPair; - protected int cur; - protected Integer floatPrecision; - private TSEncoding encoding; - private int deleteCursor = 0; - /** - * because TV list may be share with different query, each iterator has to record it's own size - */ - protected int iteSize = 0; - /** this field is effective only in the Tvlist in a RealOnlyMemChunk. */ - private List<TimeRange> deletionList; - - public Ite() { - this.iteSize = TVList.this.rowCount; + public TsBlock getTsBlock(int floatPrecision, TSEncoding encoding, List<TimeRange> deletionList) { + if (deletionList == null) { + return this.getTsBlockWithoutDeletionList(floatPrecision, encoding); } - - public Ite(int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList) { - this.floatPrecision = floatPrecision; - this.encoding = encoding; - this.iteSize = size; - this.deletionList = deletionList; - } - - @Override - public boolean hasNextTimeValuePair() { - if (hasCachedPair) { - return true; - } - - while (cur < iteSize) { - long time = getTime(cur); - if (isPointDeleted(time) || (cur + 1 < rowCount() && (time == getTime(cur + 1)))) { - cur++; - continue; - } - TimeValuePair tvPair; - tvPair = getTimeValuePair(cur, time, floatPrecision, encoding); - cur++; - if (tvPair.getValue() != null) { - cachedTimeValuePair = tvPair; - hasCachedPair = true; - return true; - } + Integer deleteCursor = 0; + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); + // Time column + TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); + for (int i = 0; i < rowCount; i++) { + if (!isPointDeleted(getTime(i), deletionList, deleteCursor)) { + timeBuilder.writeLong(this.getTime(i)); } - - return false; } - protected boolean isPointDeleted(long timestamp) { - while (deletionList != null && deleteCursor < deletionList.size()) { - if (deletionList.get(deleteCursor).contains(timestamp)) { - return true; - } else if (deletionList.get(deleteCursor).getMax() < timestamp) { - deleteCursor++; - } else { - return false; - } - } - return false; - } + // value column + ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + writeUnDeletedValuesIntoTsBlock(valueBuilder, floatPrecision, encoding, deletionList); + builder.declarePositions(rowCount); + return builder.build(); + } - @Override - public TimeValuePair nextTimeValuePair() throws IOException { - if (hasCachedPair || hasNextTimeValuePair()) { - hasCachedPair = false; - return cachedTimeValuePair; + private TsBlock getTsBlockWithoutDeletionList(int floatPrecision, TSEncoding encoding) { + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(this.getDataType())); + // Time column + TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); + for (int i = 0; i < timestamps.size() - 1; i++) { + timeBuilder.writeLongs(timestamps.get(i), ARRAY_SIZE); + } + timeBuilder.writeLongs( + timestamps.get(timestamps.size() - 1), + rowCount % ARRAY_SIZE == 0 ? ARRAY_SIZE : rowCount % ARRAY_SIZE); + + // value column + ColumnBuilder valueBuilder = builder.getColumnBuilder(0); + writeValuesIntoTsBlock(valueBuilder, floatPrecision, encoding); + builder.declarePositions(rowCount); + return builder.build(); + } + + protected abstract void writeValuesIntoTsBlock( + ColumnBuilder valueBuilder, int floatPrecision, TSEncoding encoding); + + protected abstract void writeUnDeletedValuesIntoTsBlock( + ColumnBuilder valueBuilder, + int floatPrecision, + TSEncoding encoding, + List<TimeRange> deletionList); + + protected boolean isPointDeleted( + long timestamp, List<TimeRange> deletionList, Integer deleteCursor) { + while (deleteCursor < deletionList.size()) { + if (deletionList.get(deleteCursor).contains(timestamp)) { + return true; + } else if (deletionList.get(deleteCursor).getMax() < timestamp) { + deleteCursor++; } else { - throw new IOException("no next time value pair"); + return false; } } - - @Override - public TimeValuePair currentTimeValuePair() { - return cachedTimeValuePair; - } - - @Override - public void close() throws IOException { - // Do nothing because of this is an in memory object - } + return false; } - public abstract TsBlock getTsBlock( - int floatPrecision, TSEncoding encoding, int size, List<TimeRange> deletionList); - public abstract TSDataType getDataType(); public static TVList deserialize(DataInputStream stream) throws IOException { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java index cc64db7d56..b39381cb9f 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java @@ -69,24 +69,6 @@ public class PrimitiveMemTableTest { delta = Math.pow(0.1, TSFileDescriptor.getInstance().getConfig().getFloatPrecision()); } - @Test - public void memSeriesSortIteratorTest() throws IOException { - TSDataType dataType = TSDataType.INT32; - WritableMemChunk series = - new WritableMemChunk(new MeasurementSchema("s1", dataType, TSEncoding.PLAIN)); - int count = 1000; - for (int i = 0; i < count; i++) { - series.write(i, i); - } - IPointReader it = series.getSortedTvListForQuery().getIterator(); - int i = 0; - while (it.hasNextTimeValuePair()) { - Assert.assertEquals(i, it.nextTimeValuePair().getTimestamp()); - i++; - } - Assert.assertEquals(count, i); - } - @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32;
