This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/new_vector by this push:
new dce5578 Vector -> Aligned & support query empty aligned column in
memtable
dce5578 is described below
commit dce5578e1ce7127530954d23229ebb7da44b77e1
Author: HTHou <[email protected]>
AuthorDate: Tue Nov 2 16:50:29 2021 +0800
Vector -> Aligned & support query empty aligned column in memtable
---
.../iotdb/db/engine/flush/NotifyFlushMemTable.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 12 +--
...eMemChunk.java => AlignedWritableMemChunk.java} | 61 +++++++-------
.../db/engine/memtable/IWritableMemChunk.java | 7 +-
.../db/engine/memtable/PrimitiveMemTable.java | 4 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 6 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 6 +-
.../db/engine/storagegroup/TsFileProcessor.java | 26 +++---
.../apache/iotdb/db/metadata/path/AlignedPath.java | 8 +-
.../apache/iotdb/db/metadata/path/PartialPath.java | 24 +++---
.../apache/iotdb/db/rescon/TVListAllocator.java | 20 +----
.../{VectorTVList.java => AlignedTVList.java} | 93 +++++++++++++---------
.../iotdb/db/utils/datastructure/TVList.java | 9 ++-
.../db/utils/datastructure/VectorTVListTest.java | 22 ++---
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
15 files changed, 157 insertions(+), 145 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/flush/NotifyFlushMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/flush/NotifyFlushMemTable.java
index 5bc95c9..169c22f 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/flush/NotifyFlushMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/flush/NotifyFlushMemTable.java
@@ -35,7 +35,7 @@ public class NotifyFlushMemTable extends AbstractMemTable {
}
@Override
- protected IWritableMemChunk genVectorMemSeries(IMeasurementSchema schema) {
+ protected IWritableMemChunk genAlignedMemSeries(IMeasurementSchema schema) {
return null;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 9a293fa..09656c4 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -110,7 +110,7 @@ public abstract class AbstractMemTable implements IMemTable
{
});
}
- private IWritableMemChunk createVectorMemChunkIfNotExistAndGet(
+ private IWritableMemChunk createAlignedMemChunkIfNotExistAndGet(
String deviceId, IMeasurementSchema schema) {
Map<String, IWritableMemChunk> memSeries =
memTableMap.computeIfAbsent(deviceId, k -> new HashMap<>());
@@ -122,13 +122,13 @@ public abstract class AbstractMemTable implements
IMemTable {
seriesNumber++;
totalPointsNumThreshold +=
avgSeriesPointNumThreshold *
vectorSchema.getSubMeasurementsCount();
- return genVectorMemSeries(vectorSchema);
+ return genAlignedMemSeries(vectorSchema);
});
}
protected abstract IWritableMemChunk genMemSeries(IMeasurementSchema schema);
- protected abstract IWritableMemChunk genVectorMemSeries(IMeasurementSchema
schema);
+ protected abstract IWritableMemChunk genAlignedMemSeries(IMeasurementSchema
schema);
@Override
public void insert(InsertRowPlan insertRowPlan) {
@@ -229,8 +229,8 @@ public abstract class AbstractMemTable implements IMemTable
{
@Override
public void writeAlignedRow(
String deviceId, IMeasurementSchema schema, long insertTime, Object[]
objectValue) {
- IWritableMemChunk memSeries =
createVectorMemChunkIfNotExistAndGet(deviceId, schema);
- memSeries.writeVector(insertTime, objectValue, schema);
+ IWritableMemChunk memSeries =
createAlignedMemChunkIfNotExistAndGet(deviceId, schema);
+ memSeries.writeAlignedValue(insertTime, objectValue, schema);
}
@SuppressWarnings("squid:S3776") // high Cognitive Complexity
@@ -279,7 +279,7 @@ public abstract class AbstractMemTable implements IMemTable
{
encodings.toArray(new TSEncoding[measurements.size()]),
compressionType);
IWritableMemChunk memSeries =
- createVectorMemChunkIfNotExistAndGet(
+ createAlignedMemChunkIfNotExistAndGet(
insertTabletPlan.getPrefixPath().getFullPath(), vectorSchema);
memSeries.writeVector(
insertTabletPlan.getTimes(),
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
similarity index 80%
rename from
server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
rename to
server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index c0e22d2..f155139 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -1,14 +1,14 @@
package org.apache.iotdb.db.engine.memtable;
import org.apache.iotdb.db.rescon.TVListAllocator;
+import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.db.utils.datastructure.VectorTVList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
-import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
@@ -20,25 +20,25 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class VectorWritableMemChunk implements IWritableMemChunk {
+public class AlignedWritableMemChunk implements IWritableMemChunk {
private IMeasurementSchema schema;
- private VectorTVList list;
- private Map<String, Integer> vectorIdIndexMap;
+ private AlignedTVList list;
+ private Map<String, Integer> alignedMeasurementIndexMap;
private static final String UNSUPPORTED_TYPE = "Unsupported data type:";
- private static final Logger LOGGER =
LoggerFactory.getLogger(VectorWritableMemChunk.class);
+ private static final Logger LOGGER =
LoggerFactory.getLogger(AlignedWritableMemChunk.class);
- public VectorWritableMemChunk(VectorMeasurementSchema schema) {
+ public AlignedWritableMemChunk(VectorMeasurementSchema schema) {
this.schema = schema;
- vectorIdIndexMap = new HashMap<>();
+ alignedMeasurementIndexMap = new HashMap<>();
for (int i = 0; i < schema.getSubMeasurementsCount(); i++) {
- vectorIdIndexMap.put(schema.getSubMeasurementsList().get(i), i);
+ alignedMeasurementIndexMap.put(schema.getSubMeasurementsList().get(i),
i);
}
this.list =
TVListAllocator.getInstance().allocate(schema.getSubMeasurementsTSDataTypeList());
}
public boolean containsMeasurement(String measurementId) {
- return vectorIdIndexMap.containsKey(measurementId);
+ return alignedMeasurementIndexMap.containsKey(measurementId);
}
@Override
@@ -72,8 +72,8 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putVector(long t, Object[] v, int[] columnOrder) {
- list.putVector(t, v, columnOrder);
+ public void putAlignedValue(long t, Object[] v, int[] columnOrder) {
+ list.putAlignedValue(t, v, columnOrder);
}
@Override
@@ -107,9 +107,9 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putVectors(
+ public void putAlignedValues(
long[] t, Object[] v, BitMap[] bitMaps, int[] columnOrder, int start,
int end) {
- list.putVectors(t, v, bitMaps, columnOrder, start, end);
+ list.putAlignedValues(t, v, bitMaps, columnOrder, start, end);
}
@Override
@@ -118,9 +118,9 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void writeVector(long insertTime, Object[] objectValue,
IMeasurementSchema schema) {
+ public void writeAlignedValue(long insertTime, Object[] objectValue,
IMeasurementSchema schema) {
int[] columnOrder = checkColumnOrder(schema);
- putVector(insertTime, objectValue, columnOrder);
+ putAlignedValue(insertTime, objectValue, columnOrder);
}
@Override
@@ -138,7 +138,7 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
int start,
int end) {
int[] columnOrder = checkColumnOrder(schema);
- putVectors(times, valueList, bitMaps, columnOrder, start, end);
+ putAlignedValues(times, valueList, bitMaps, columnOrder, start, end);
}
private int[] checkColumnOrder(IMeasurementSchema schema) {
@@ -146,7 +146,7 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
List<String> measurementIdList = vectorSchema.getSubMeasurementsList();
int[] columnOrder = new int[measurementIdList.size()];
for (int i = 0; i < measurementIdList.size(); i++) {
- columnOrder[i] = vectorIdIndexMap.get(measurementIdList.get(i));
+ columnOrder[i] =
alignedMeasurementIndexMap.get(measurementIdList.get(i));
}
return columnOrder;
}
@@ -158,7 +158,7 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
@Override
public long count() {
- return list.size() * vectorIdIndexMap.size();
+ return list.size() * alignedMeasurementIndexMap.size();
}
@Override
@@ -182,7 +182,8 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
List<Integer> columnIndexList = new ArrayList<>();
List<TSDataType> dataTypeList = new ArrayList<>();
for (IMeasurementSchema measurementSchema : schemaList) {
-
columnIndexList.add(vectorIdIndexMap.getOrDefault(measurementSchema.getMeasurementId(),
-1));
+ columnIndexList.add(
+
alignedMeasurementIndexMap.getOrDefault(measurementSchema.getMeasurementId(),
-1));
dataTypeList.add(measurementSchema.getType());
}
return list.getTvListByColumnIndex(columnIndexList, dataTypeList);
@@ -224,29 +225,29 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
@Override
public void encode(IChunkWriter chunkWriter) {
- List<Integer> timeDuplicatedVectorRowIndexList = null;
+ List<Integer> timeDuplicateAlignedRowIndexList = null;
for (int sortedRowIndex = 0; sortedRowIndex < list.size();
sortedRowIndex++) {
long time = list.getTime(sortedRowIndex);
// skip duplicated data
if ((sortedRowIndex + 1 < list.size() && (time ==
list.getTime(sortedRowIndex + 1)))) {
// record the time duplicated row index list for vector type
- if (timeDuplicatedVectorRowIndexList == null) {
- timeDuplicatedVectorRowIndexList = new ArrayList<>();
-
timeDuplicatedVectorRowIndexList.add(list.getValueIndex(sortedRowIndex));
+ if (timeDuplicateAlignedRowIndexList == null) {
+ timeDuplicateAlignedRowIndexList = new ArrayList<>();
+
timeDuplicateAlignedRowIndexList.add(list.getValueIndex(sortedRowIndex));
}
- timeDuplicatedVectorRowIndexList.add(list.getValueIndex(sortedRowIndex
+ 1));
+ timeDuplicateAlignedRowIndexList.add(list.getValueIndex(sortedRowIndex
+ 1));
continue;
}
List<TSDataType> dataTypes = list.getTsDataTypes();
int originRowIndex = list.getValueIndex(sortedRowIndex);
for (int columnIndex = 0; columnIndex < dataTypes.size(); columnIndex++)
{
// write the time duplicated rows
- if (timeDuplicatedVectorRowIndexList != null
- && !timeDuplicatedVectorRowIndexList.isEmpty()) {
+ if (timeDuplicateAlignedRowIndexList != null
+ && !timeDuplicateAlignedRowIndexList.isEmpty()) {
originRowIndex =
list.getValidRowIndexForTimeDuplicatedRows(
- timeDuplicatedVectorRowIndexList, columnIndex);
+ timeDuplicateAlignedRowIndexList, columnIndex);
}
boolean isNull = list.isValueMarked(originRowIndex, columnIndex);
switch (dataTypes.get(columnIndex)) {
@@ -273,13 +274,13 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
break;
default:
LOGGER.error(
- "VectorWritableMemChunk does not support data type: {}",
+ "AlignedWritableMemChunk does not support data type: {}",
dataTypes.get(columnIndex));
break;
}
}
chunkWriter.write(time);
- timeDuplicatedVectorRowIndexList = null;
+ timeDuplicateAlignedRowIndexList = null;
}
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index 28b4026..17f3bc3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -41,7 +41,7 @@ public interface IWritableMemChunk {
void putBoolean(long t, boolean v);
- void putVector(long t, Object[] v, int[] columnOrder);
+ void putAlignedValue(long t, Object[] v, int[] columnOrder);
void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end);
@@ -55,11 +55,12 @@ public interface IWritableMemChunk {
void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end);
- void putVectors(long[] t, Object[] v, BitMap[] bitMaps, int[] columnOrder,
int start, int end);
+ void putAlignedValues(
+ long[] t, Object[] v, BitMap[] bitMaps, int[] columnOrder, int start,
int end);
void write(long insertTime, Object objectValue);
- void writeVector(long insertTime, Object[] objectValue, IMeasurementSchema
schema);
+ void writeAlignedValue(long insertTime, Object[] objectValue,
IMeasurementSchema schema);
/**
* write data in the range [start, end). Null value in the valueList will be
replaced by the
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
index 71cd460..6915967 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
@@ -43,8 +43,8 @@ public class PrimitiveMemTable extends AbstractMemTable {
}
@Override
- protected IWritableMemChunk genVectorMemSeries(IMeasurementSchema schema) {
- return new VectorWritableMemChunk((VectorMeasurementSchema) schema);
+ protected IWritableMemChunk genAlignedMemSeries(IMeasurementSchema schema) {
+ return new AlignedWritableMemChunk((VectorMeasurementSchema) schema);
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index 3638f58..5e064af 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -72,7 +72,7 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void writeVector(long insertTime, Object[] objectValue,
IMeasurementSchema schema) {
+ public void writeAlignedValue(long insertTime, Object[] objectValue,
IMeasurementSchema schema) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
list.getDataType());
}
@@ -151,7 +151,7 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putVector(long t, Object[] v, int[] columnOrder) {
+ public void putAlignedValue(long t, Object[] v, int[] columnOrder) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
schema.getType());
}
@@ -186,7 +186,7 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putVectors(
+ public void putAlignedValues(
long[] t, Object[] v, BitMap[] bitMaps, int[] columnOrder, int start,
int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
schema.getType());
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
index 18d26e3..f0fdbeb 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
@@ -145,7 +145,7 @@ public class ReadOnlyMemChunk {
}
/**
- * The constructor for VECTOR type.
+ * The constructor for Aligned type.
*
* @param schema VectorMeasurementSchema
* @param tvList VectorTvList
@@ -164,11 +164,11 @@ public class ReadOnlyMemChunk {
this.chunkPointReader =
tvList.getIterator(floatPrecision, encoding, chunkDataSize,
deletionList);
- initVectorChunkMeta((VectorMeasurementSchema) schema);
+ initAlignedChunkMeta((VectorMeasurementSchema) schema);
}
@SuppressWarnings("squid:S3776") // high Cognitive Complexity
- private void initVectorChunkMeta(VectorMeasurementSchema schema)
+ private void initAlignedChunkMeta(VectorMeasurementSchema schema)
throws IOException, QueryProcessException {
Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR);
IChunkMetadata timeChunkMetadata =
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index dbdc859..b50c904 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -48,8 +48,8 @@ import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.db.utils.QueryUtils;
+import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.db.utils.datastructure.VectorTVList;
import org.apache.iotdb.db.writelog.WALFlushListener;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
@@ -212,7 +212,7 @@ public class TsFileProcessor {
long[] memIncrements = null;
if (enableMemControl) {
if (insertRowPlan.isAligned()) {
- memIncrements = checkMemCostAndAddToTspInfoForVector(insertRowPlan);
+ memIncrements = checkAlignedMemCostAndAddToTspInfo(insertRowPlan);
} else {
memIncrements = checkMemCostAndAddToTspInfo(insertRowPlan);
}
@@ -278,7 +278,7 @@ public class TsFileProcessor {
try {
if (enableMemControl) {
if (insertTabletPlan.isAligned()) {
- memIncrements =
checkMemCostAndAddToTspInfoForVector(insertTabletPlan, start, end);
+ memIncrements = checkAlignedMemCostAndAddToTsp(insertTabletPlan,
start, end);
} else {
memIncrements = checkMemCostAndAddToTspInfo(insertTabletPlan, start,
end);
}
@@ -372,7 +372,7 @@ public class TsFileProcessor {
}
@SuppressWarnings("squid:S3776") // high Cognitive Complexity
- private long[] checkMemCostAndAddToTspInfoForVector(InsertRowPlan
insertRowPlan)
+ private long[] checkAlignedMemCostAndAddToTspInfo(InsertRowPlan
insertRowPlan)
throws WriteProcessException {
// memory of increased PrimitiveArray and TEXT values, e.g., add a
long[128], add 128*8
long memTableIncrement = 0L;
@@ -384,13 +384,13 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(
insertRowPlan.getMeasurements()[0],
insertRowPlan.getDataTypes()[0]);
- memTableIncrement +=
VectorTVList.vectorTvListArrayMemSize(insertRowPlan.getDataTypes());
+ memTableIncrement +=
AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes());
} else {
// here currentChunkPointNum >= 1
int currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, null);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ?
VectorTVList.vectorTvListArrayMemSize(insertRowPlan.getDataTypes())
+ ?
AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes())
: 0;
}
for (int i = 0; i < insertRowPlan.getDataTypes().length; i++) {
@@ -433,7 +433,7 @@ public class TsFileProcessor {
return memIncrements;
}
- private long[] checkMemCostAndAddToTspInfoForVector(
+ private long[] checkAlignedMemCostAndAddToTsp(
InsertTabletPlan insertTabletPlan, int start, int end) throws
WriteProcessException {
if (start >= end) {
return new long[] {0, 0, 0};
@@ -442,7 +442,7 @@ public class TsFileProcessor {
String deviceId = insertTabletPlan.getPrefixPath().getFullPath();
- updateVectorMemCost(
+ updateAlignedMemCost(
insertTabletPlan.getDataTypes(),
deviceId,
insertTabletPlan.getMeasurements()[0],
@@ -494,7 +494,7 @@ public class TsFileProcessor {
}
}
- private void updateVectorMemCost(
+ private void updateAlignedMemCost(
TSDataType[] dataTypes,
String deviceId,
String measurementId,
@@ -509,19 +509,21 @@ public class TsFileProcessor {
dataTypes.length * ChunkMetadata.calculateRamSize(measurementId,
dataTypes[0]);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * VectorTVList.vectorTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
} else {
int currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, null);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * VectorTVList.vectorTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
} else {
int acquireArray =
(end - start - 1 + (currentChunkPointNum %
PrimitiveArrayManager.ARRAY_SIZE))
/ PrimitiveArrayManager.ARRAY_SIZE;
memIncrements[0] +=
- acquireArray == 0 ? 0 : acquireArray *
VectorTVList.vectorTvListArrayMemSize(dataTypes);
+ acquireArray == 0
+ ? 0
+ : acquireArray *
AlignedTVList.alignedTvListArrayMemSize(dataTypes);
}
}
// TEXT data size
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index a8398b8..ef42c21 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -19,8 +19,8 @@
package org.apache.iotdb.db.metadata.path;
+import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
-import org.apache.iotdb.db.engine.memtable.VectorWritableMemChunk;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -269,7 +269,7 @@ public class AlignedPath extends PartialPath {
timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1);
timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
timeTimeSeriesMetadata.setMeasurementId("");
- timeTimeSeriesMetadata.setTSDataType(TSDataType.INT64);
+ timeTimeSeriesMetadata.setTSDataType(TSDataType.VECTOR);
Statistics<? extends Serializable> timeStatistics =
Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType());
@@ -325,8 +325,8 @@ public class AlignedPath extends PartialPath {
if (!memTableMap.containsKey(getDevice())) {
return null;
}
- VectorWritableMemChunk vectorMemChunk =
- ((VectorWritableMemChunk)
memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER));
+ AlignedWritableMemChunk vectorMemChunk =
+ ((AlignedWritableMemChunk)
memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER));
boolean containsMeasurement = false;
for (String measurement : measurementList) {
if (vectorMemChunk.containsMeasurement(measurement)) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
index 62f5774..f620d27 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
@@ -18,17 +18,6 @@
*/
package org.apache.iotdb.db.metadata.path;
-import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -48,9 +37,22 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
+import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
+
/**
* A prefix path, suffix path or fullPath generated from SQL. Usually used in
the IoTDB server
* module
diff --git
a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
index 06902e6..6d0c4e7 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
@@ -24,8 +24,8 @@ import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.JMXService;
import org.apache.iotdb.db.service.ServiceType;
+import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.db.utils.datastructure.VectorTVList;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import java.util.ArrayDeque;
@@ -57,24 +57,12 @@ public class TVListAllocator implements
TVListAllocatorMBean, IService {
return list != null ? list : TVList.newList(dataType);
}
- public synchronized VectorTVList allocate(List<TSDataType> dataTypes) {
- return VectorTVList.newVectorList(dataTypes);
+ public synchronized AlignedTVList allocate(List<TSDataType> dataTypes) {
+ return AlignedTVList.newAlignedList(dataTypes);
}
- /** For non-vector types. */
- public synchronized void release(TSDataType dataType, TVList list) {
- list.clear();
- if (dataType != TSDataType.VECTOR) {
- tvListCache.get(list.getDataType()).add(list);
- }
- }
-
- /** For VECTOR type only. */
public synchronized void release(TVList list) {
- list.clear();
- if (list.getDataType() != TSDataType.VECTOR) {
- tvListCache.get(list.getDataType()).add(list);
- }
+ list.clear(tvListCache);
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
similarity index 92%
rename from
server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
rename to
server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index cabec52..5576f08 100644
---
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
+++
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -32,12 +32,14 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Queue;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
-public class VectorTVList extends TVList {
+public class AlignedTVList extends TVList {
- // data types of this vector
+ // data types of this aligned tvlist
private List<TSDataType> dataTypes;
// data type list -> list of TVList, add 1 when expanded -> primitive array
of basic type
@@ -58,7 +60,7 @@ public class VectorTVList extends TVList {
private int pivotIndex;
- VectorTVList(List<TSDataType> types) {
+ AlignedTVList(List<TSDataType> types) {
super();
indices = new ArrayList<>(types.size());
dataTypes = types;
@@ -68,13 +70,13 @@ public class VectorTVList extends TVList {
}
}
- public static VectorTVList newVectorList(List<TSDataType> datatypes) {
- return new VectorTVList(datatypes);
+ public static AlignedTVList newAlignedList(List<TSDataType> datatypes) {
+ return new AlignedTVList(datatypes);
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
- public void putVector(long timestamp, Object[] value, int[] columnOrder) {
+ public void putAlignedValue(long timestamp, Object[] value, int[]
columnOrder) {
checkExpansion();
int arrayIndex = size / ARRAY_SIZE;
int elementIndex = size % ARRAY_SIZE;
@@ -130,21 +132,21 @@ public class VectorTVList extends TVList {
int arrayIndex = index / ARRAY_SIZE;
int elementIndex = index % ARRAY_SIZE;
int valueIndex = indices.get(arrayIndex)[elementIndex];
- return getVectorByValueIndex(valueIndex, null);
+ return getAlignedValueByValueIndex(valueIndex, null);
}
- public TsPrimitiveType getVector(List<Integer> timeDuplicatedIndexList) {
+ public TsPrimitiveType getAlignedValue(List<Integer>
timeDuplicatedIndexList) {
int[] validIndexesForTimeDuplicatedRows = new int[values.size()];
for (int i = 0; i < values.size(); i++) {
validIndexesForTimeDuplicatedRows[i] =
getValidRowIndexForTimeDuplicatedRows(timeDuplicatedIndexList, i);
}
- return getVectorByValueIndex(
+ return getAlignedValueByValueIndex(
timeDuplicatedIndexList.get(timeDuplicatedIndexList.size() - 1),
validIndexesForTimeDuplicatedRows);
}
- private TsPrimitiveType getVectorByValueIndex(
+ private TsPrimitiveType getAlignedValueByValueIndex(
int valueIndex, int[] validIndexesForTimeDuplicatedRows) {
if (valueIndex >= size) {
throw new ArrayIndexOutOfBoundsException(valueIndex);
@@ -237,13 +239,13 @@ public class VectorTVList extends TVList {
}
}
}
- VectorTVList vectorTvList = new VectorTVList(types);
- vectorTvList.timestamps = this.timestamps;
- vectorTvList.indices = this.indices;
- vectorTvList.values = values;
- vectorTvList.bitMaps = bitMaps;
- vectorTvList.size = this.size;
- return vectorTvList;
+ AlignedTVList alignedTvList = new AlignedTVList(types);
+ alignedTvList.timestamps = this.timestamps;
+ alignedTvList.indices = this.indices;
+ alignedTvList.values = values;
+ alignedTvList.bitMaps = bitMaps;
+ alignedTvList.size = this.size;
+ return alignedTvList;
}
private void generateEmptyColumn(
@@ -277,12 +279,13 @@ public class VectorTVList extends TVList {
bitMap.markAll();
columnBitMaps.add(bitMap);
}
+ // values.size() is the index of column
+ bitMaps.set(values.size(), columnBitMaps);
values.add(columnValue);
- bitMaps.add(columnBitMaps);
}
/**
- * Get the int value at the given position in VectorTvList.
+ * Get the int value at the given position in AlignedTvList.
*
* @param rowIndex value index inside this column
* @param columnIndex index of the column
@@ -434,8 +437,8 @@ public class VectorTVList extends TVList {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
- public VectorTVList clone() {
- VectorTVList cloneList = new VectorTVList(dataTypes);
+ public AlignedTVList clone() {
+ AlignedTVList cloneList = new AlignedTVList(dataTypes);
cloneAs(cloneList);
for (int[] indicesArray : indices) {
cloneList.indices.add(cloneIndex(indicesArray));
@@ -662,7 +665,7 @@ public class VectorTVList extends TVList {
@Override
public TimeValuePair getTimeValuePairForTimeDuplicatedRows(
List<Integer> indexList, long time, Integer floatPrecision, TSEncoding
encoding) {
- return new TimeValuePair(time, getVector(indexList));
+ return new TimeValuePair(time, getAlignedValue(indexList));
}
@Override
@@ -675,7 +678,7 @@ public class VectorTVList extends TVList {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
- public void putVectors(
+ public void putAlignedValues(
long[] time, Object[] value, BitMap[] bitMaps, int[] columnOrder, int
start, int end) {
checkExpansion();
int idx = start;
@@ -793,12 +796,12 @@ public class VectorTVList extends TVList {
}
/**
- * Get the single vectorTVList array size by give types.
+ * Get the single alignedTVList array size by give types.
*
* @param types the types in the vector
- * @return VectorTvListArrayMemSize
+ * @return AlignedTvListArrayMemSize
*/
- public static long vectorTvListArrayMemSize(TSDataType[] types) {
+ public static long alignedTvListArrayMemSize(TSDataType[] types) {
long size = 0;
// time size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
@@ -812,24 +815,36 @@ public class VectorTVList extends TVList {
}
@Override
+ public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
+ size = 0;
+ sorted = true;
+ minTime = Long.MAX_VALUE;
+ clearTime();
+ clearSortedTime();
+
+ clearValue();
+ clearSortedValue();
+ }
+
+ @Override
@TestOnly
public IPointReader getIterator() {
- return new VectorIte();
+ return new AlignedIte();
}
@Override
public IPointReader getIterator(
int floatPrecision, TSEncoding encoding, int size, List<TimeRange>
deletionList) {
- return new VectorIte(floatPrecision, encoding, size, deletionList);
+ return new AlignedIte(floatPrecision, encoding, size, deletionList);
}
- private class VectorIte extends Ite {
+ private class AlignedIte extends Ite {
- public VectorIte() {
+ public AlignedIte() {
super();
}
- public VectorIte(
+ public AlignedIte(
int floatPrecision, TSEncoding encoding, int size, List<TimeRange>
deletionList) {
super(floatPrecision, encoding, size, deletionList);
}
@@ -840,24 +855,24 @@ public class VectorTVList extends TVList {
return true;
}
- List<Integer> timeDuplicatedVectorRowIndexList = null;
+ List<Integer> timeDuplicatedAlignedRowIndexList = null;
while (cur < iteSize) {
long time = getTime(cur);
if (isPointDeleted(time) || (cur + 1 < size() && (time == getTime(cur
+ 1)))) {
- if (timeDuplicatedVectorRowIndexList == null) {
- timeDuplicatedVectorRowIndexList = new ArrayList<>();
- timeDuplicatedVectorRowIndexList.add(getValueIndex(cur));
+ if (timeDuplicatedAlignedRowIndexList == null) {
+ timeDuplicatedAlignedRowIndexList = new ArrayList<>();
+ timeDuplicatedAlignedRowIndexList.add(getValueIndex(cur));
}
- timeDuplicatedVectorRowIndexList.add(getValueIndex(cur + 1));
+ timeDuplicatedAlignedRowIndexList.add(getValueIndex(cur + 1));
cur++;
continue;
}
TimeValuePair tvPair;
- if (timeDuplicatedVectorRowIndexList != null) {
+ if (timeDuplicatedAlignedRowIndexList != null) {
tvPair =
getTimeValuePairForTimeDuplicatedRows(
- timeDuplicatedVectorRowIndexList, time, floatPrecision,
encoding);
- timeDuplicatedVectorRowIndexList = null;
+ timeDuplicatedAlignedRowIndexList, time, floatPrecision,
encoding);
+ timeDuplicatedAlignedRowIndexList = null;
} else {
tvPair = getTimeValuePair(cur, time, floatPrecision, encoding);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 3fe1e3a..a77544a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.utils.BitMap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
@@ -140,7 +142,7 @@ public abstract class TVList {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public void putVector(long time, Object[] value, int[] columnOrder) {
+ public void putAlignedValue(long time, Object[] value, int[] columnOrder) {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
@@ -168,7 +170,7 @@ public abstract class TVList {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public void putVectors(
+ public void putAlignedValues(
long[] time, Object[] value, BitMap[] bitMaps, int[] columnOrder, int
start, int end) {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
@@ -276,7 +278,7 @@ public abstract class TVList {
cloneList.minTime = minTime;
}
- public void clear() {
+ public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
size = 0;
sorted = true;
minTime = Long.MAX_VALUE;
@@ -285,6 +287,7 @@ public abstract class TVList {
clearValue();
clearSortedValue();
+ tvListCache.get(getDataType()).add(this);
}
protected void clearTime() {
diff --git
a/server/src/test/java/org/apache/iotdb/db/utils/datastructure/VectorTVListTest.java
b/server/src/test/java/org/apache/iotdb/db/utils/datastructure/VectorTVListTest.java
index f0e7835..14aa8b2 100644
---
a/server/src/test/java/org/apache/iotdb/db/utils/datastructure/VectorTVListTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/utils/datastructure/VectorTVListTest.java
@@ -37,7 +37,7 @@ public class VectorTVListTest {
for (int i = 0; i < 5; i++) {
dataTypes.add(TSDataType.INT64);
}
- VectorTVList tvList = new VectorTVList(dataTypes);
+ AlignedTVList tvList = new AlignedTVList(dataTypes);
for (long i = 0; i < 1000; i++) {
Object[] value = new Object[5];
int[] columnOrder = new int[5];
@@ -45,7 +45,7 @@ public class VectorTVListTest {
value[j] = i;
columnOrder[j] = j;
}
- tvList.putVector(i, value, columnOrder);
+ tvList.putAlignedValue(i, value, columnOrder);
}
for (int i = 0; i < tvList.size; i++) {
StringBuilder builder = new StringBuilder("[");
@@ -68,7 +68,7 @@ public class VectorTVListTest {
dataTypes.add(TSDataType.FLOAT);
dataTypes.add(TSDataType.DOUBLE);
dataTypes.add(TSDataType.TEXT);
- VectorTVList tvList = new VectorTVList(dataTypes);
+ AlignedTVList tvList = new AlignedTVList(dataTypes);
for (int i = 1000; i >= 0; i--) {
Object[] value = new Object[6];
value[0] = false;
@@ -81,7 +81,7 @@ public class VectorTVListTest {
for (int j = 0; j < 6; j++) {
columnOrder[j] = j;
}
- tvList.putVector(i, value, columnOrder);
+ tvList.putAlignedValue(i, value, columnOrder);
}
tvList.sort();
for (int i = 0; i < tvList.size; i++) {
@@ -99,7 +99,7 @@ public class VectorTVListTest {
for (int i = 0; i < 5; i++) {
dataTypes.add(TSDataType.INT64);
}
- VectorTVList tvList = new VectorTVList(dataTypes);
+ AlignedTVList tvList = new AlignedTVList(dataTypes);
long[][] vectorArray = new long[5][1001];
List<Long> timeList = new ArrayList<>();
int[] columnOrder = new int[5];
@@ -111,7 +111,7 @@ public class VectorTVListTest {
}
}
- tvList.putVectors(
+ tvList.putAlignedValues(
ArrayUtils.toPrimitive(timeList.toArray(new Long[0])),
vectorArray,
null,
@@ -131,7 +131,7 @@ public class VectorTVListTest {
dataTypes.add(TSDataType.INT64);
bitMaps[i] = new BitMap(1001);
}
- VectorTVList tvList = new VectorTVList(dataTypes);
+ AlignedTVList tvList = new AlignedTVList(dataTypes);
long[][] vectorArray = new long[5][1001];
int[] columnOrder = new int[5];
List<Long> timeList = new ArrayList<>();
@@ -146,7 +146,7 @@ public class VectorTVListTest {
}
}
- tvList.putVectors(
+ tvList.putAlignedValues(
ArrayUtils.toPrimitive(timeList.toArray(new Long[0])),
vectorArray,
bitMaps,
@@ -169,7 +169,7 @@ public class VectorTVListTest {
dataTypes.add(TSDataType.INT64);
bitMaps[i] = new BitMap(1001);
}
- VectorTVList tvList = new VectorTVList(dataTypes);
+ AlignedTVList tvList = new AlignedTVList(dataTypes);
long[][] vectorArray = new long[5][1001];
int[] columnOrder = new int[5];
List<Long> timeList = new ArrayList<>();
@@ -184,7 +184,7 @@ public class VectorTVListTest {
}
}
- tvList.putVectors(
+ tvList.putAlignedValues(
ArrayUtils.toPrimitive(timeList.toArray(new Long[0])),
vectorArray,
bitMaps,
@@ -192,7 +192,7 @@ public class VectorTVListTest {
0,
1000);
- VectorTVList clonedTvList = tvList.clone();
+ AlignedTVList clonedTvList = tvList.clone();
for (long i = 0; i < tvList.size; i++) {
Assert.assertEquals(tvList.getTime((int) i), clonedTvList.getTime((int)
i));
Assert.assertEquals(
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 01e6024..dad4fdd 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -74,7 +74,7 @@ public class ChunkMetadata implements Accountable,
IChunkMetadata {
private boolean isSeq = true;
private boolean isClosed;
private String filePath;
-
+
// 0x80 for time chunk, 0x40 for value chunk, 0x00 for common chunk
private byte mask;
