This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/new_vector by this push:
new 554392c support query a non-exsit measurement in vector
554392c is described below
commit 554392c594cf877e2f26d35ff96749dc13896086
Author: HTHou <[email protected]>
AuthorDate: Tue Nov 2 11:45:20 2021 +0800
support query a non-exsit measurement in vector
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 2 +-
.../db/engine/memtable/IWritableMemChunk.java | 2 +-
.../db/engine/memtable/VectorWritableMemChunk.java | 10 ++--
.../iotdb/db/engine/memtable/WritableMemChunk.java | 2 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 32 -----------
.../apache/iotdb/db/metadata/path/AlignedPath.java | 10 ++--
.../iotdb/db/utils/datastructure/TVList.java | 2 +-
.../iotdb/db/utils/datastructure/VectorTVList.java | 64 +++++++++++++++++++---
8 files changed, 72 insertions(+), 52 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index b719f8c..9a293fa 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -121,7 +121,7 @@ public abstract class AbstractMemTable implements IMemTable
{
k -> {
seriesNumber++;
totalPointsNumThreshold +=
- avgSeriesPointNumThreshold +
vectorSchema.getSubMeasurementsCount();
+ avgSeriesPointNumThreshold *
vectorSchema.getSubMeasurementsCount();
return genVectorMemSeries(vectorSchema);
});
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index 1d27b0d..28b4026 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -104,7 +104,7 @@ public interface IWritableMemChunk {
* @param measurementList the measurementList to be queried
* @return sorted tv list
*/
- TVList getSortedTvListForQuery(List<String> measurementList);
+ TVList getSortedTvListForQuery(List<IMeasurementSchema> schemaList);
/**
* served for flush requests. The logic is just same as
getSortedTVListForQuery, but without add
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
index dd16b8f..c60d6f8 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
@@ -175,15 +175,17 @@ public class VectorWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public TVList getSortedTvListForQuery(List<String> measurementList) {
+ public TVList getSortedTvListForQuery(List<IMeasurementSchema> schemaList) {
sortTVList();
// increase reference count
list.increaseReferenceCount();
List<Integer> columnIndexList = new ArrayList<>();
- for (String measurement : measurementList) {
- columnIndexList.add(vectorIdIndexMap.get(measurement));
+ List<TSDataType> dataTypeList = new ArrayList<>();
+ for (IMeasurementSchema measurementSchema : schemaList) {
+
columnIndexList.add(vectorIdIndexMap.getOrDefault(measurementSchema.getMeasurementId(),
-1));
+ dataTypeList.add(measurementSchema.getType());
}
- return list.getTvListByColumnIndex(columnIndexList);
+ return list.getTvListByColumnIndex(columnIndexList, dataTypeList);
}
private void sortTVList() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index 8dcfc07..3638f58 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -200,7 +200,7 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public synchronized TVList getSortedTvListForQuery(List<String>
measurementList) {
+ public synchronized TVList getSortedTvListForQuery(List<IMeasurementSchema>
measurementSchema) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
list.getDataType());
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
index 568fe39..18d26e3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
@@ -206,38 +206,6 @@ public class ReadOnlyMemChunk {
cachedMetaData = vectorChunkMetadata;
}
- // When query one measurement in a Vector, the timeValuePair is not a vector
type
- private void updateValueStatisticsForSingleColumn(
- IMeasurementSchema schema, Statistics[] valueStatistics, TimeValuePair
timeValuePair)
- throws QueryProcessException {
- switch (schema.getSubMeasurementsTSDataTypeList().get(0)) {
- case BOOLEAN:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(),
timeValuePair.getValue().getBoolean());
- break;
- case TEXT:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(),
timeValuePair.getValue().getBinary());
- break;
- case FLOAT:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- break;
- case INT32:
- valueStatistics[0].update(timeValuePair.getTimestamp(),
timeValuePair.getValue().getInt());
- break;
- case INT64:
- valueStatistics[0].update(timeValuePair.getTimestamp(),
timeValuePair.getValue().getLong());
- break;
- case DOUBLE:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(),
timeValuePair.getValue().getDouble());
- break;
- default:
- throw new QueryProcessException("Unsupported data type:" + dataType);
- }
- }
-
private void updateValueStatistics(
IMeasurementSchema schema, Statistics[] valueStatistics, TimeValuePair
timeValuePair)
throws QueryProcessException {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index 877a8fb..a8398b8 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -321,22 +321,24 @@ public class AlignedPath extends PartialPath {
public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable(
Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange>
deletionList)
throws QueryProcessException, IOException {
+ // check If Memtable Contains this path
if (!memTableMap.containsKey(getDevice())) {
return null;
}
VectorWritableMemChunk vectorMemChunk =
((VectorWritableMemChunk)
memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER));
- List<String> validMeasurementList = new ArrayList<>();
+ boolean containsMeasurement = false;
for (String measurement : measurementList) {
if (vectorMemChunk.containsMeasurement(measurement)) {
- validMeasurementList.add(measurement);
+ containsMeasurement = true;
+ break;
}
}
- if (validMeasurementList.isEmpty()) {
+ if (!containsMeasurement) {
return null;
}
// get sorted tv list is synchronized so different query can get right
sorted list reference
- TVList vectorTvListCopy =
vectorMemChunk.getSortedTvListForQuery(validMeasurementList);
+ TVList vectorTvListCopy =
vectorMemChunk.getSortedTvListForQuery(schemaList);
int curSize = vectorTvListCopy.size();
return new ReadOnlyMemChunk(getMeasurementSchema(), vectorTvListCopy,
curSize, deletionList);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 1a46af4..3fe1e3a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -201,7 +201,7 @@ public abstract class TVList {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public TVList getTvListByColumnIndex(List<Integer> columnIndexList) {
+ public TVList getTvListByColumnIndex(List<Integer> columnIndexList,
List<TSDataType> dataTypes) {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
index 24ea2cb..cabec52 100644
---
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
+++
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
@@ -207,21 +207,34 @@ public class VectorTVList extends TVList {
}
@Override
- public TVList getTvListByColumnIndex(List<Integer> columns) {
+ public TVList getTvListByColumnIndex(List<Integer> columnIndex,
List<TSDataType> dataTypes) {
List<TSDataType> types = new ArrayList<>();
List<List<Object>> values = new ArrayList<>();
List<List<BitMap>> bitMaps = null;
- for (int column : columns) {
- types.add(this.dataTypes.get(column));
- values.add(this.values.get(column));
- if (this.bitMaps != null && this.bitMaps.get(column) != null) {
+ for (int i = 0; i < columnIndex.size(); i++) {
+ // columnIndex == -1 means querying a non-exist column, generate empty
column here
+ if (columnIndex.get(i) == -1) {
+ types.add(dataTypes.get(i));
+ // use bitmap to mark as null value
if (bitMaps == null) {
- bitMaps = new ArrayList<>(columns.size());
- for (int i = 0; i < columns.size(); i++) {
+ bitMaps = new ArrayList<>(columnIndex.size());
+ for (int j = 0; j < columnIndex.size(); j++) {
bitMaps.add(null);
}
}
- bitMaps.set(columns.indexOf(column), this.bitMaps.get(column));
+ generateEmptyColumn(dataTypes.get(i), values, bitMaps);
+ } else {
+ types.add(this.dataTypes.get(columnIndex.get(i)));
+ values.add(this.values.get(columnIndex.get(i)));
+ if (this.bitMaps != null && this.bitMaps.get(columnIndex.get(i)) !=
null) {
+ if (bitMaps == null) {
+ bitMaps = new ArrayList<>(columnIndex.size());
+ for (int j = 0; j < columnIndex.size(); j++) {
+ bitMaps.add(null);
+ }
+ }
+ bitMaps.set(i, this.bitMaps.get(columnIndex.get(i)));
+ }
}
}
VectorTVList vectorTvList = new VectorTVList(types);
@@ -233,6 +246,41 @@ public class VectorTVList extends TVList {
return vectorTvList;
}
+ private void generateEmptyColumn(
+ TSDataType dataType, List<List<Object>> values, List<List<BitMap>>
bitMaps) {
+ List<Object> columnValue = new ArrayList<>();
+ List<BitMap> columnBitMaps = new ArrayList<>();
+ for (int i = 0; i < timestamps.size(); i++) {
+ switch (dataType) {
+ case TEXT:
+ columnValue.add(new Binary[ARRAY_SIZE]);
+ break;
+ case FLOAT:
+ columnValue.add(new float[ARRAY_SIZE]);
+ break;
+ case INT32:
+ columnValue.add(new int[ARRAY_SIZE]);
+ break;
+ case INT64:
+ columnValue.add(new long[ARRAY_SIZE]);
+ break;
+ case DOUBLE:
+ columnValue.add(new double[ARRAY_SIZE]);
+ break;
+ case BOOLEAN:
+ columnValue.add(new boolean[ARRAY_SIZE]);
+ break;
+ default:
+ break;
+ }
+ BitMap bitMap = new BitMap(ARRAY_SIZE);
+ bitMap.markAll();
+ columnBitMaps.add(bitMap);
+ }
+ values.add(columnValue);
+ bitMaps.add(columnBitMaps);
+ }
+
/**
* Get the int value at the given position in VectorTvList.
*
