This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch new_vector in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 45cc05964105bd6bff548499c6156a7bd7e41ba0 Merge: b763e96 50fb89c Author: JackieTien97 <[email protected]> AuthorDate: Mon Nov 1 19:59:36 2021 +0800 resolve conflicts .../iotdb/db/engine/memtable/AbstractMemTable.java | 65 +++-------------- .../db/engine/memtable/IWritableMemChunk.java | 4 +- .../db/engine/memtable/PrimitiveMemTable.java | 3 +- .../db/engine/memtable/VectorWritableMemChunk.java | 17 ++++- .../iotdb/db/engine/memtable/WritableMemChunk.java | 2 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 13 ++-- .../db/engine/storagegroup/TsFileResource.java | 43 ++++++----- .../apache/iotdb/db/metadata/path/AlignedPath.java | 83 ++++++++++++++++------ .../iotdb/db/metadata/path/MeasurementPath.java | 54 +++++++++++--- .../apache/iotdb/db/metadata/path/PartialPath.java | 42 +++++++---- .../iotdb/db/query/reader/chunk/MemPageReader.java | 2 +- .../iotdb/db/query/reader/series/SeriesReader.java | 4 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 6 +- .../write/schema/VectorMeasurementSchema.java | 2 +- 14 files changed, 196 insertions(+), 144 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index a71f62e,bee5e10..b719f8c --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@@ -123,12 -115,13 +115,14 @@@ public abstract class AbstractMemTable Map<String, IWritableMemChunk> memSeries = memTableMap.computeIfAbsent(deviceId, k -> new HashMap<>()); + VectorMeasurementSchema vectorSchema = (VectorMeasurementSchema) schema; return memSeries.computeIfAbsent( - schema.getMeasurementId(), + vectorSchema.getMeasurementId(), k -> { seriesNumber++; - totalPointsNumThreshold += avgSeriesPointNumThreshold + schema.getSubMeasurementsCount(); - return genVectorMemSeries(schema); - totalPointsNumThreshold += avgSeriesPointNumThreshold + vectorSchema.getSubMeasurementsCount(); ++ totalPointsNumThreshold += ++ avgSeriesPointNumThreshold + vectorSchema.getSubMeasurementsCount(); + return genVectorMemSeries(vectorSchema); }); } diff --cc server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java index a827a1f,616efd5..71cd460 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java @@@ -20,7 -20,7 +20,8 @@@ package org.apache.iotdb.db.engine.memtable; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; + import java.util.HashMap; import java.util.Map; diff --cc server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java index f15e25c,694ecd9..568fe39 --- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java @@@ -23,9 -23,9 +23,9 @@@ import org.apache.iotdb.db.query.reader import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; ++import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; - import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; @@@ -33,7 -33,7 +33,8 @@@ import org.apache.iotdb.tsfile.read.Tim import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.reader.IPointReader; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index a2b532e,11a9538..c973886 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@@ -62,6 -50,26 +45,25 @@@ import org.apache.iotdb.tsfile.utils.Re import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.File; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; -import java.io.Serializable; + import java.nio.file.FileAlreadyExistsException; + import java.nio.file.Files; + import java.nio.file.Paths; + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Objects; + import java.util.Random; + import java.util.Set; + + import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR; + import static org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator.getTsFileName; + import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; + @SuppressWarnings("java:S1135") // ignore todos public class TsFileResource { @@@ -108,7 -116,8 +110,6 @@@ private boolean isSeq; - private Map<String, Integer> holderMap = new HashMap<>(); -- /** * Chunk metadata list of unsealed tsfile. Only be set in a temporal TsFileResource in a query * process. @@@ -211,7 -222,113 +212,6 @@@ this.timeIndexType = 1; } - /** - * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't - * have chunkMetadata, but query will use these, so we need to generate it for them. - */ - @SuppressWarnings("squid:S3776") // high Cognitive Complexity - private void generateTimeSeriesMetadata() throws IOException { - TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); - timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); - timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); - - if (!(chunkMetadataList == null || chunkMetadataList.isEmpty())) { - timeTimeSeriesMetadata.setMeasurementId(chunkMetadataList.get(0).getMeasurementUid()); - TSDataType dataType = chunkMetadataList.get(0).getDataType(); - timeTimeSeriesMetadata.setTSDataType(dataType); - } else if (!(readOnlyMemChunk == null || readOnlyMemChunk.isEmpty())) { - timeTimeSeriesMetadata.setMeasurementId(readOnlyMemChunk.get(0).getMeasurementUid()); - TSDataType dataType = readOnlyMemChunk.get(0).getDataType(); - timeTimeSeriesMetadata.setTSDataType(dataType); - } - if (timeTimeSeriesMetadata.getTSDataType() != null) { - if (timeTimeSeriesMetadata.getTSDataType() == TSDataType.VECTOR) { - Statistics<? extends Serializable> timeStatistics = - Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); - - List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>(); - - if (!(chunkMetadataList == null || chunkMetadataList.isEmpty())) { - VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetadataList.get(0); - for (IChunkMetadata valueChunkMetadata : - vectorChunkMetadata.getValueChunkMetadataList()) { - TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); - valueMetadata.setOffsetOfChunkMetaDataList(-1); - valueMetadata.setDataSizeOfChunkMetaDataList(-1); - valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementUid()); - valueMetadata.setTSDataType(valueChunkMetadata.getDataType()); - valueTimeSeriesMetadataList.add(valueMetadata); - valueMetadata.setStatistics( - Statistics.getStatsByType(valueChunkMetadata.getDataType())); - } - } else if (!(readOnlyMemChunk == null || readOnlyMemChunk.isEmpty())) { - VectorChunkMetadata vectorChunkMetadata = - (VectorChunkMetadata) readOnlyMemChunk.get(0).getChunkMetaData(); - for (IChunkMetadata valueChunkMetadata : - vectorChunkMetadata.getValueChunkMetadataList()) { - TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); - valueMetadata.setOffsetOfChunkMetaDataList(-1); - valueMetadata.setDataSizeOfChunkMetaDataList(-1); - valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementUid()); - valueMetadata.setTSDataType(valueChunkMetadata.getDataType()); - valueTimeSeriesMetadataList.add(valueMetadata); - valueMetadata.setStatistics( - Statistics.getStatsByType(valueChunkMetadata.getDataType())); - } - } - - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetadata; - timeStatistics.mergeStatistics( - vectorChunkMetadata.getTimeChunkMetadata().getStatistics()); - for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - valueTimeSeriesMetadataList - .get(i) - .getStatistics() - .mergeStatistics( - vectorChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); - } - } - - for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { - if (!memChunk.isEmpty()) { - VectorChunkMetadata vectorChunkMetadata = - (VectorChunkMetadata) memChunk.getChunkMetaData(); - timeStatistics.mergeStatistics( - vectorChunkMetadata.getTimeChunkMetadata().getStatistics()); - for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - valueTimeSeriesMetadataList - .get(i) - .getStatistics() - .mergeStatistics( - vectorChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); - } - } - } - timeTimeSeriesMetadata.setStatistics(timeStatistics); - timeSeriesMetadata = - new VectorTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); - } else { - Statistics<? extends Serializable> seriesStatistics = - Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); - // flush chunkMetadataList one by one - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); - } - - for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { - if (!memChunk.isEmpty()) { - seriesStatistics.mergeStatistics(memChunk.getChunkMetaData().getStatistics()); - } - } - timeTimeSeriesMetadata.setStatistics(seriesStatistics); - this.timeSeriesMetadata = timeTimeSeriesMetadata; - } - } else { - this.timeSeriesMetadata = null; - } - } -- public synchronized void serialize() throws IOException { try (OutputStream outputStream = fsFactory.getBufferedOutputStream(file + RESOURCE_SUFFIX + TEMP_SUFFIX)) { @@@ -585,11 -702,6 +585,10 @@@ return timeSeriesMetadata; } - public void setTimeSeriesMetadata( - ITimeSeriesMetadata timeSeriesMetadata) { ++ public void setTimeSeriesMetadata(ITimeSeriesMetadata timeSeriesMetadata) { + this.timeSeriesMetadata = timeSeriesMetadata; + } + public void setUpgradedResources(List<TsFileResource> upgradedResources) { this.upgradedResources = upgradedResources; } diff --cc server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java index ce3ce57,2a09598..877a8fb --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java @@@ -34,17 -30,23 +30,28 @@@ import org.apache.iotdb.db.query.contex import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.series.AlignedSeriesReader; import org.apache.iotdb.db.utils.TestOnly; - import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; - import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; + import org.apache.iotdb.db.utils.datastructure.TVList; -import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; ++import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; ++import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; + import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; + import java.io.IOException; ++import java.io.Serializable; + import java.util.ArrayList; + import java.util.Collections; + import java.util.List; + import java.util.Map; + import java.util.Objects; + import java.util.Set; + /** * VectorPartialPath represents a vector's fullPath. It not only contains the full path of vector's * own name, but also has subSensorsList which contain all the fullPath of vector's sub sensors. @@@ -234,73 -241,26 +246,98 @@@ public class AlignedPath extends Partia } @Override - public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, - List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) ++ public TsFileResource createTsFileResource( ++ List<ReadOnlyMemChunk> readOnlyMemChunk, ++ List<IChunkMetadata> chunkMetadataList, ++ TsFileResource originTsFileResource) + throws IOException { - TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); - tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); ++ TsFileResource tsFileResource = ++ new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); ++ tsFileResource.setTimeSeriesMetadata( ++ generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); + return tsFileResource; + } + + /** + * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't + * have chunkMetadata, but query will use these, so we need to generate it for them. + */ - private AlignedTimeSeriesMetadata generateTimeSeriesMetadata(List<ReadOnlyMemChunk> readOnlyMemChunk, - List<IChunkMetadata> chunkMetadataList) throws IOException { ++ private AlignedTimeSeriesMetadata generateTimeSeriesMetadata( ++ List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList) ++ throws IOException { + TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); + timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); + timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); + timeTimeSeriesMetadata.setMeasurementId(""); + timeTimeSeriesMetadata.setTSDataType(TSDataType.INT64); + - + Statistics<? extends Serializable> timeStatistics = + Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); + + // init each value time series meta + List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>(); + for (IMeasurementSchema valueChunkMetadata : schemaList) { + TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); + valueMetadata.setOffsetOfChunkMetaDataList(-1); + valueMetadata.setDataSizeOfChunkMetaDataList(-1); + valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementId()); + valueMetadata.setTSDataType(valueChunkMetadata.getType()); - valueMetadata.setStatistics( - Statistics.getStatsByType(valueChunkMetadata.getType())); ++ valueMetadata.setStatistics(Statistics.getStatsByType(valueChunkMetadata.getType())); + valueTimeSeriesMetadataList.add(valueMetadata); + } + + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; - timeStatistics.mergeStatistics( - alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); ++ timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); + for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { + valueTimeSeriesMetadataList + .get(i) + .getStatistics() + .mergeStatistics( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); + } + } + + for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { + if (!memChunk.isEmpty()) { + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) memChunk.getChunkMetaData(); - timeStatistics.mergeStatistics( - alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); ++ timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); + for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { + valueTimeSeriesMetadataList + .get(i) + .getStatistics() + .mergeStatistics( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); + } + } + } + timeTimeSeriesMetadata.setStatistics(timeStatistics); + + return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + } ++ ++ @Override + public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( + Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange> deletionList) + throws QueryProcessException, IOException { + if (!memTableMap.containsKey(getDevice())) { + return null; + } + VectorWritableMemChunk vectorMemChunk = + ((VectorWritableMemChunk) memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER)); + List<String> validMeasurementList = new ArrayList<>(); + for (String measurement : measurementList) { + if (vectorMemChunk.containsMeasurement(measurement)) { + validMeasurementList.add(measurement); + } + } + if (validMeasurementList.isEmpty()) { + return null; + } + // get sorted tv list is synchronized so different query can get right sorted list reference + TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(validMeasurementList); + int curSize = vectorTvListCopy.size(); + return new ReadOnlyMemChunk(getMeasurementSchema(), vectorTvListCopy, curSize, deletionList); + } } diff --cc server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java index 4fb5b07,bd6d129..7d384b9 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java @@@ -31,13 -29,17 +29,21 @@@ import org.apache.iotdb.db.query.contex import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.series.SeriesReader; import org.apache.iotdb.db.utils.TestOnly; + import org.apache.iotdb.db.utils.datastructure.TVList; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; + import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + import java.io.IOException; ++import java.io.Serializable; + import java.util.List; + import java.util.Map; + import java.util.Set; + public class MeasurementPath extends PartialPath { private IMeasurementSchema measurementSchema; @@@ -147,39 -149,25 +153,67 @@@ } @Override - public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, - List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) ++ public TsFileResource createTsFileResource( ++ List<ReadOnlyMemChunk> readOnlyMemChunk, ++ List<IChunkMetadata> chunkMetadataList, ++ TsFileResource originTsFileResource) + throws IOException { - TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); - tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); ++ TsFileResource tsFileResource = ++ new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); ++ tsFileResource.setTimeSeriesMetadata( ++ generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); + return tsFileResource; + } + + /** + * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't + * have chunkMetadata, but query will use these, so we need to generate it for them. + */ - private TimeseriesMetadata generateTimeSeriesMetadata(List<ReadOnlyMemChunk> readOnlyMemChunk, - List<IChunkMetadata> chunkMetadataList) throws IOException { ++ private TimeseriesMetadata generateTimeSeriesMetadata( ++ List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList) ++ throws IOException { + TimeseriesMetadata timeSeriesMetadata = new TimeseriesMetadata(); + timeSeriesMetadata.setMeasurementId(measurementSchema.getMeasurementId()); + timeSeriesMetadata.setTSDataType(measurementSchema.getType()); + timeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); + timeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); + + Statistics<? extends Serializable> seriesStatistics = + Statistics.getStatsByType(timeSeriesMetadata.getTSDataType()); + // flush chunkMetadataList one by one + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + } + + for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { + if (!memChunk.isEmpty()) { + seriesStatistics.mergeStatistics(memChunk.getChunkMetaData().getStatistics()); + } + } + timeSeriesMetadata.setStatistics(seriesStatistics); + return timeSeriesMetadata; + } ++ ++ @Override + public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( + Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange> deletionList) + throws QueryProcessException, IOException { + // check If Memtable Contains this path + if (!memTableMap.containsKey(getDevice()) + || !memTableMap.get(getDevice()).containsKey(getMeasurement())) { + return null; + } + IWritableMemChunk memChunk = memTableMap.get(getDevice()).get(getMeasurement()); + // get sorted tv list is synchronized so different query can get right sorted list reference + TVList chunkCopy = memChunk.getSortedTvListForQuery(); + int curSize = chunkCopy.size(); + return new ReadOnlyMemChunk( + getMeasurement(), + measurementSchema.getType(), + measurementSchema.getEncodingType(), + chunkCopy, + measurementSchema.getProps(), + curSize, + deletionList); + } } diff --cc server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java index f19a4fe,648049c..a360c68 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java @@@ -39,11 -31,12 +31,13 @@@ import org.apache.iotdb.db.query.filter import org.apache.iotdb.db.query.reader.series.SeriesReader; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; + import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -389,9 -394,14 +395,21 @@@ public class PartialPath extends Path i throw new UnsupportedOperationException("Should call exact sub class!"); } - public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, - List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) ++ public TsFileResource createTsFileResource( ++ List<ReadOnlyMemChunk> readOnlyMemChunk, ++ List<IChunkMetadata> chunkMetadataList, ++ TsFileResource originTsFileResource) + throws IOException { + throw new UnsupportedOperationException("Should call exact sub class!"); + } + /** + * get the ReadOnlyMemChunk from the given MemTable. + * + * @return ReadOnlyMemChunk + */ + public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( + Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange> deletionList) + throws QueryProcessException, IOException { + throw new UnsupportedOperationException("Should call exact sub class!"); + } } diff --cc server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java index 5c42d81,f123ad7..0070f94 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java @@@ -18,8 -18,8 +18,8 @@@ */ package org.apache.iotdb.db.query.reader.chunk; - import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; + import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TimeValuePair; diff --cc server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 36d3680,5dda91a..c27a2d3 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@@ -31,10 -31,10 +31,10 @@@ import org.apache.iotdb.db.query.reader import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.db.utils.TestOnly; - import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; - import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; + import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; + import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorTimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TimeValuePair; diff --cc server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 96ffe0d,f4e1b67..854645b --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@@ -29,12 -29,12 +29,12 @@@ import org.apache.iotdb.db.query.reader import org.apache.iotdb.db.query.reader.chunk.MemChunkReader; import org.apache.iotdb.db.query.reader.chunk.metadata.DiskChunkMetadataLoader; import org.apache.iotdb.db.query.reader.chunk.metadata.MemChunkMetadataLoader; ++import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; ++import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; - import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; - import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorTimeSeriesMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; @@@ -42,8 -42,8 +42,8 @@@ import org.apache.iotdb.tsfile.read.con import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; - import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; +import org.apache.iotdb.tsfile.read.reader.chunk.AlignedChunkReader; + import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; -import org.apache.iotdb.tsfile.read.reader.chunk.VectorChunkReader; import java.io.IOException; import java.util.ArrayList; diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java index e8a5c55,e6bae26..d2e97cd --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java @@@ -22,9 -22,9 +22,9 @@@ package org.apache.iotdb.tsfile.write.s import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder; ++import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; - import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
