This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch new_vector in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b763e96ec77b6c2903eec80e8e9068600a9eeb0a Author: JackieTien97 <[email protected]> AuthorDate: Mon Nov 1 19:53:45 2021 +0800 remove if instance of --- .../db/engine/querycontext/ReadOnlyMemChunk.java | 4 +- .../db/engine/storagegroup/TsFileProcessor.java | 2 +- .../db/engine/storagegroup/TsFileResource.java | 160 ++++----------------- .../apache/iotdb/db/metadata/path/AlignedPath.java | 90 +++++++++++- .../iotdb/db/metadata/path/MeasurementPath.java | 48 ++++++- .../apache/iotdb/db/metadata/path/PartialPath.java | 29 ++-- .../iotdb/db/query/reader/chunk/MemPageReader.java | 8 +- .../iotdb/db/query/reader/series/SeriesReader.java | 20 +-- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 44 +++--- ...hunkMetadata.java => AlignedChunkMetadata.java} | 4 +- ...etadata.java => AlignedTimeSeriesMetadata.java} | 6 +- ...torChunkReader.java => AlignedChunkReader.java} | 16 +-- ...ectorPageReader.java => AlignedPageReader.java} | 4 +- .../write/schema/VectorMeasurementSchema.java | 4 +- 14 files changed, 227 insertions(+), 212 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java index bdd9e3b..f15e25c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java @@ -25,7 +25,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; @@ -203,7 +203,7 @@ public class ReadOnlyMemChunk { valueStatistic.setEmpty(isEmpty()); } IChunkMetadata vectorChunkMetadata = - new VectorChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); vectorChunkMetadata.setChunkLoader(new MemChunkLoader(this)); vectorChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = vectorChunkMetadata; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 0c7c227..dbdc859 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -1281,7 +1281,7 @@ public class TsFileProcessor { // get in memory data if (!readOnlyMemChunks.isEmpty() || !chunkMetadataList.isEmpty()) { tsfileResourcesForQuery.add( - new TsFileResource(readOnlyMemChunks, chunkMetadataList, tsFileResource)); + fullPath.createTsFileResource(readOnlyMemChunks, chunkMetadataList, tsFileResource)); } } catch (QueryProcessException e) { logger.error( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index 11a9538..a2b532e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -18,6 +18,24 @@ */ package org.apache.iotdb.db.engine.storagegroup; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR; +import static org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator.getTsFileName; +import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.modification.ModificationFile; @@ -36,40 +54,14 @@ import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorTimeSeriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.utils.FilePathUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Serializable; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Random; -import java.util.Set; - -import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR; -import static org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator.getTsFileName; -import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; - @SuppressWarnings("java:S1135") // ignore todos public class TsFileResource { @@ -112,11 +104,10 @@ public class TsFileResource { private TsFileLock tsFileLock = new TsFileLock(); - private Random random = new Random(); + private final Random random = new Random(); private boolean isSeq; - private Map<String, Integer> holderMap = new HashMap<>(); /** * Chunk metadata list of unsealed tsfile. Only be set in a temporal TsFileResource in a query @@ -173,7 +164,6 @@ public class TsFileResource { this.isMerging = other.isMerging; this.chunkMetadataList = other.chunkMetadataList; this.readOnlyMemChunk = other.readOnlyMemChunk; - generateTimeSeriesMetadata(); this.tsFileLock = other.tsFileLock; this.fsFactory = other.fsFactory; this.maxPlanIndex = other.maxPlanIndex; @@ -211,7 +201,6 @@ public class TsFileResource { this.readOnlyMemChunk = readOnlyMemChunk; this.originTsFileResource = originTsFileResource; this.version = originTsFileResource.version; - generateTimeSeriesMetadata(); } @TestOnly @@ -222,112 +211,6 @@ public class TsFileResource { this.timeIndexType = 1; } - /** - * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't - * have chunkMetadata, but query will use these, so we need to generate it for them. - */ - @SuppressWarnings("squid:S3776") // high Cognitive Complexity - private void generateTimeSeriesMetadata() throws IOException { - TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); - timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); - timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); - - if (!(chunkMetadataList == null || chunkMetadataList.isEmpty())) { - timeTimeSeriesMetadata.setMeasurementId(chunkMetadataList.get(0).getMeasurementUid()); - TSDataType dataType = chunkMetadataList.get(0).getDataType(); - timeTimeSeriesMetadata.setTSDataType(dataType); - } else if (!(readOnlyMemChunk == null || readOnlyMemChunk.isEmpty())) { - timeTimeSeriesMetadata.setMeasurementId(readOnlyMemChunk.get(0).getMeasurementUid()); - TSDataType dataType = readOnlyMemChunk.get(0).getDataType(); - timeTimeSeriesMetadata.setTSDataType(dataType); - } - if (timeTimeSeriesMetadata.getTSDataType() != null) { - if (timeTimeSeriesMetadata.getTSDataType() == TSDataType.VECTOR) { - Statistics<? extends Serializable> timeStatistics = - Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); - - List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>(); - - if (!(chunkMetadataList == null || chunkMetadataList.isEmpty())) { - VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetadataList.get(0); - for (IChunkMetadata valueChunkMetadata : - vectorChunkMetadata.getValueChunkMetadataList()) { - TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); - valueMetadata.setOffsetOfChunkMetaDataList(-1); - valueMetadata.setDataSizeOfChunkMetaDataList(-1); - valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementUid()); - valueMetadata.setTSDataType(valueChunkMetadata.getDataType()); - valueTimeSeriesMetadataList.add(valueMetadata); - valueMetadata.setStatistics( - Statistics.getStatsByType(valueChunkMetadata.getDataType())); - } - } else if (!(readOnlyMemChunk == null || readOnlyMemChunk.isEmpty())) { - VectorChunkMetadata vectorChunkMetadata = - (VectorChunkMetadata) readOnlyMemChunk.get(0).getChunkMetaData(); - for (IChunkMetadata valueChunkMetadata : - vectorChunkMetadata.getValueChunkMetadataList()) { - TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); - valueMetadata.setOffsetOfChunkMetaDataList(-1); - valueMetadata.setDataSizeOfChunkMetaDataList(-1); - valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementUid()); - valueMetadata.setTSDataType(valueChunkMetadata.getDataType()); - valueTimeSeriesMetadataList.add(valueMetadata); - valueMetadata.setStatistics( - Statistics.getStatsByType(valueChunkMetadata.getDataType())); - } - } - - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetadata; - timeStatistics.mergeStatistics( - vectorChunkMetadata.getTimeChunkMetadata().getStatistics()); - for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - valueTimeSeriesMetadataList - .get(i) - .getStatistics() - .mergeStatistics( - vectorChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); - } - } - - for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { - if (!memChunk.isEmpty()) { - VectorChunkMetadata vectorChunkMetadata = - (VectorChunkMetadata) memChunk.getChunkMetaData(); - timeStatistics.mergeStatistics( - vectorChunkMetadata.getTimeChunkMetadata().getStatistics()); - for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - valueTimeSeriesMetadataList - .get(i) - .getStatistics() - .mergeStatistics( - vectorChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); - } - } - } - timeTimeSeriesMetadata.setStatistics(timeStatistics); - timeSeriesMetadata = - new VectorTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); - } else { - Statistics<? extends Serializable> seriesStatistics = - Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); - // flush chunkMetadataList one by one - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); - } - - for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { - if (!memChunk.isEmpty()) { - seriesStatistics.mergeStatistics(memChunk.getChunkMetaData().getStatistics()); - } - } - timeTimeSeriesMetadata.setStatistics(seriesStatistics); - this.timeSeriesMetadata = timeTimeSeriesMetadata; - } - } else { - this.timeSeriesMetadata = null; - } - } public synchronized void serialize() throws IOException { try (OutputStream outputStream = @@ -702,6 +585,11 @@ public class TsFileResource { return timeSeriesMetadata; } + public void setTimeSeriesMetadata( + ITimeSeriesMetadata timeSeriesMetadata) { + this.timeSeriesMetadata = timeSeriesMetadata; + } + public void setUpgradedResources(List<TsFileResource> upgradedResources) { this.upgradedResources = upgradedResources; } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java index 4d3640d..ce3ce57 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java @@ -19,25 +19,32 @@ package org.apache.iotdb.db.metadata.path; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; +import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.series.AlignedSeriesReader; import org.apache.iotdb.db.utils.TestOnly; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; - /** * VectorPartialPath represents a vector's fullPath. It not only contains the full path of vector's * own name, but also has subSensorsList which contain all the fullPath of vector's sub sensors. @@ -225,4 +232,75 @@ public class AlignedPath extends PartialPath { valueFilter, ascending); } + + @Override + public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, + List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) + throws IOException { + TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); + tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); + return tsFileResource; + } + + /** + * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't + * have chunkMetadata, but query will use these, so we need to generate it for them. + */ + private AlignedTimeSeriesMetadata generateTimeSeriesMetadata(List<ReadOnlyMemChunk> readOnlyMemChunk, + List<IChunkMetadata> chunkMetadataList) throws IOException { + TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); + timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); + timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); + timeTimeSeriesMetadata.setMeasurementId(""); + timeTimeSeriesMetadata.setTSDataType(TSDataType.INT64); + + + Statistics<? extends Serializable> timeStatistics = + Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType()); + + // init each value time series meta + List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>(); + for (IMeasurementSchema valueChunkMetadata : schemaList) { + TimeseriesMetadata valueMetadata = new TimeseriesMetadata(); + valueMetadata.setOffsetOfChunkMetaDataList(-1); + valueMetadata.setDataSizeOfChunkMetaDataList(-1); + valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementId()); + valueMetadata.setTSDataType(valueChunkMetadata.getType()); + valueMetadata.setStatistics( + Statistics.getStatsByType(valueChunkMetadata.getType())); + valueTimeSeriesMetadataList.add(valueMetadata); + } + + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; + timeStatistics.mergeStatistics( + alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); + for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { + valueTimeSeriesMetadataList + .get(i) + .getStatistics() + .mergeStatistics( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); + } + } + + for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { + if (!memChunk.isEmpty()) { + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) memChunk.getChunkMetaData(); + timeStatistics.mergeStatistics( + alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); + for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { + valueTimeSeriesMetadataList + .get(i) + .getStatistics() + .mergeStatistics( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); + } + } + } + timeTimeSeriesMetadata.setStatistics(timeStatistics); + + return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java index b328990..4fb5b07 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java @@ -18,21 +18,26 @@ */ package org.apache.iotdb.db.metadata.path; +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.Set; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; +import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.series.SeriesReader; import org.apache.iotdb.db.utils.TestOnly; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; -import java.util.List; -import java.util.Set; - public class MeasurementPath extends PartialPath { private IMeasurementSchema measurementSchema; @@ -140,4 +145,41 @@ public class MeasurementPath extends PartialPath { valueFilter, ascending); } + + @Override + public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, + List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) + throws IOException { + TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource); + tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList)); + return tsFileResource; + } + + /** + * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't + * have chunkMetadata, but query will use these, so we need to generate it for them. + */ + private TimeseriesMetadata generateTimeSeriesMetadata(List<ReadOnlyMemChunk> readOnlyMemChunk, + List<IChunkMetadata> chunkMetadataList) throws IOException { + TimeseriesMetadata timeSeriesMetadata = new TimeseriesMetadata(); + timeSeriesMetadata.setMeasurementId(measurementSchema.getMeasurementId()); + timeSeriesMetadata.setTSDataType(measurementSchema.getType()); + timeSeriesMetadata.setOffsetOfChunkMetaDataList(-1); + timeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); + + Statistics<? extends Serializable> seriesStatistics = + Statistics.getStatsByType(timeSeriesMetadata.getTSDataType()); + // flush chunkMetadataList one by one + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + } + + for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { + if (!memChunk.isEmpty()) { + seriesStatistics.mergeStatistics(memChunk.getChunkMetaData().getStatistics()); + } + } + timeSeriesMetadata.setStatistics(seriesStatistics); + return timeSeriesMetadata; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java index fde7705..f19a4fe 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java @@ -18,7 +18,18 @@ */ package org.apache.iotdb.db.metadata.path; +import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; +import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -28,24 +39,14 @@ import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.series.SeriesReader; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; - -import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; -import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; - /** * A prefix path, suffix path or fullPath generated from SQL. Usually used in the IoTDB server * module @@ -387,4 +388,10 @@ public class PartialPath extends Path implements Comparable<Path> { boolean ascending) { throw new UnsupportedOperationException("Should call exact sub class!"); } + + public TsFileResource createTsFileResource(List<ReadOnlyMemChunk> readOnlyMemChunk, + List<IChunkMetadata> chunkMetadataList, TsFileResource originTsFileResource) + throws IOException { + throw new UnsupportedOperationException("Should call exact sub class!"); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java index f123ad7..5c42d81 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.query.reader.chunk; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TimeValuePair; @@ -48,10 +48,10 @@ public class MemPageReader implements IPageReader { @Override public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { TSDataType dataType; - if (chunkMetadata instanceof VectorChunkMetadata - && ((VectorChunkMetadata) chunkMetadata).getValueChunkMetadataList().size() == 1) { + if (chunkMetadata instanceof AlignedChunkMetadata + && ((AlignedChunkMetadata) chunkMetadata).getValueChunkMetadataList().size() == 1) { dataType = - ((VectorChunkMetadata) chunkMetadata).getValueChunkMetadataList().get(0).getDataType(); + ((AlignedChunkMetadata) chunkMetadata).getValueChunkMetadataList().get(0).getDataType(); } else { dataType = chunkMetadata.getDataType(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 5dda91a..36d3680 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -33,8 +33,8 @@ import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorTimeSeriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TimeValuePair; @@ -43,7 +43,7 @@ import org.apache.iotdb.tsfile.read.common.BatchDataFactory; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter; import org.apache.iotdb.tsfile.read.reader.IPageReader; -import org.apache.iotdb.tsfile.read.reader.page.VectorPageReader; +import org.apache.iotdb.tsfile.read.reader.page.AlignedPageReader; import java.io.IOException; import java.io.Serializable; @@ -276,10 +276,10 @@ public class SeriesReader { } Statistics currentFileStatistics(int index) throws IOException { - if (!(firstTimeSeriesMetadata instanceof VectorTimeSeriesMetadata)) { + if (!(firstTimeSeriesMetadata instanceof AlignedTimeSeriesMetadata)) { throw new IOException("Can only get statistics by index from vectorTimeSeriesMetaData"); } - return ((VectorTimeSeriesMetadata) firstTimeSeriesMetadata).getStatistics(index); + return ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getStatistics(index); } boolean currentFileModified() throws IOException { @@ -409,10 +409,10 @@ public class SeriesReader { } Statistics currentChunkStatistics(int index) throws IOException { - if (!(firstChunkMetadata instanceof VectorChunkMetadata)) { + if (!(firstChunkMetadata instanceof AlignedChunkMetadata)) { throw new IOException("Can only get statistics by index from vectorChunkMetaData"); } - return ((VectorChunkMetadata) firstChunkMetadata).getStatistics(index); + return ((AlignedChunkMetadata) firstChunkMetadata).getStatistics(index); } boolean currentChunkModified() throws IOException { @@ -1071,7 +1071,7 @@ public class SeriesReader { } public boolean isVectorPageReader() { - return data instanceof VectorPageReader; + return data instanceof AlignedPageReader; } Statistics getStatistics() { @@ -1079,10 +1079,10 @@ public class SeriesReader { } Statistics getStatistics(int index) throws IOException { - if (!(data instanceof VectorPageReader)) { + if (!(data instanceof AlignedPageReader)) { throw new IOException("Can only get statistics by index from VectorPageReader"); } - return ((VectorPageReader) data).getStatistics(index); + return ((AlignedPageReader) data).getStatistics(index); } BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index f4e1b67..96ffe0d 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -33,8 +33,8 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorTimeSeriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; @@ -43,7 +43,7 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; -import org.apache.iotdb.tsfile.read.reader.chunk.VectorChunkReader; +import org.apache.iotdb.tsfile.read.reader.chunk.AlignedChunkReader; import java.io.IOException; import java.util.ArrayList; @@ -154,14 +154,14 @@ public class FileLoaderUtils { * [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2]) * @param allSensors all sensors belonging to this device that appear in query */ - public static VectorTimeSeriesMetadata loadTimeSeriesMetadata( + public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata( TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter, Set<String> allSensors) throws IOException { - VectorTimeSeriesMetadata vectorTimeSeriesMetadata = null; + AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // If the tsfile is closed, we need to load from tsfile if (resource.isClosed()) { if (!resource.getTsFile().exists()) { @@ -194,35 +194,35 @@ public class FileLoaderUtils { .setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, subPath, context, filter)); } - vectorTimeSeriesMetadata = - new VectorTimeSeriesMetadata( + alignedTimeSeriesMetadata = + new AlignedTimeSeriesMetadata( timeSeriesMetadata.get(0), timeSeriesMetadata.subList(1, timeSeriesMetadata.size())); } } else { // if the tsfile is unclosed, we just get it directly from TsFileResource - vectorTimeSeriesMetadata = (VectorTimeSeriesMetadata) resource.getTimeSeriesMetadata(); - if (vectorTimeSeriesMetadata != null) { - vectorTimeSeriesMetadata.setChunkMetadataLoader( + alignedTimeSeriesMetadata = (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(); + if (alignedTimeSeriesMetadata != null) { + alignedTimeSeriesMetadata.setChunkMetadataLoader( new MemChunkMetadataLoader(resource, vectorPath, context, filter)); } } - if (vectorTimeSeriesMetadata != null) { + if (alignedTimeSeriesMetadata != null) { List<Modification> pathModifications = context.getPathModifications(resource.getModFile(), vectorPath); - vectorTimeSeriesMetadata.getTimeseriesMetadata().setModified(!pathModifications.isEmpty()); - if (vectorTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() - > vectorTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { + alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(!pathModifications.isEmpty()); + if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() + > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { return null; } if (filter != null && !filter.satisfyStartEndTime( - vectorTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(), - vectorTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) { + alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(), + alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) { return null; } List<TimeseriesMetadata> valueTimeSeriesMetadataList = - vectorTimeSeriesMetadata.getValueTimeseriesMetadataList(); + alignedTimeSeriesMetadata.getValueTimeseriesMetadataList(); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { pathModifications = context.getPathModifications( @@ -230,7 +230,7 @@ public class FileLoaderUtils { valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty()); } } - return vectorTimeSeriesMetadata; + return alignedTimeSeriesMetadata; } /** @@ -265,10 +265,10 @@ public class FileLoaderUtils { chunk.setFromOldFile(chunkMetaData.isFromOldTsFile()); chunkReader = new ChunkReader(chunk, timeFilter); } else { - VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetaData; - Chunk timeChunk = vectorChunkMetadata.getTimeChunk(); - List<Chunk> valueChunkList = vectorChunkMetadata.getValueChunkList(); - chunkReader = new VectorChunkReader(timeChunk, valueChunkList, timeFilter); + AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; + Chunk timeChunk = alignedChunkMetadata.getTimeChunk(); + List<Chunk> valueChunkList = alignedChunkMetadata.getValueChunkList(); + chunkReader = new AlignedChunkReader(timeChunk, valueChunkList, timeFilter); } } return chunkReader.loadPageReaderList(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java similarity index 98% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java index 2bfda53..f42bb3c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java @@ -29,14 +29,14 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; -public class VectorChunkMetadata implements IChunkMetadata { +public class AlignedChunkMetadata implements IChunkMetadata { // ChunkMetadata for time column private final IChunkMetadata timeChunkMetadata; // ChunkMetadata for all subSensors in the vector private final List<IChunkMetadata> valueChunkMetadataList; - public VectorChunkMetadata( + public AlignedChunkMetadata( IChunkMetadata timeChunkMetadata, List<IChunkMetadata> valueChunkMetadataList) { this.timeChunkMetadata = timeChunkMetadata; this.valueChunkMetadataList = valueChunkMetadataList; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java similarity index 95% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java index c3d727f..b2a21ec 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java @@ -25,14 +25,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -public class VectorTimeSeriesMetadata implements ITimeSeriesMetadata { +public class AlignedTimeSeriesMetadata implements ITimeSeriesMetadata { // TimeSeriesMetadata for time column private final TimeseriesMetadata timeseriesMetadata; // TimeSeriesMetadata for all subSensors in the vector private final List<TimeseriesMetadata> valueTimeseriesMetadataList; - public VectorTimeSeriesMetadata( + public AlignedTimeSeriesMetadata( TimeseriesMetadata timeseriesMetadata, List<TimeseriesMetadata> valueTimeseriesMetadataList) { this.timeseriesMetadata = timeseriesMetadata; this.valueTimeseriesMetadataList = valueTimeseriesMetadataList; @@ -105,7 +105,7 @@ public class VectorTimeSeriesMetadata implements ITimeSeriesMetadata { for (List<IChunkMetadata> chunkMetadata : valueChunkMetadataList) { chunkMetadataList.add(chunkMetadata.get(i)); } - res.add(new VectorChunkMetadata(timeChunkMetadata.get(i), chunkMetadataList)); + res.add(new AlignedChunkMetadata(timeChunkMetadata.get(i), chunkMetadataList)); } return res; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java similarity index 95% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java index 3cb62c8..8afd569 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java @@ -33,7 +33,7 @@ import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; -import org.apache.iotdb.tsfile.read.reader.page.VectorPageReader; +import org.apache.iotdb.tsfile.read.reader.page.AlignedPageReader; import java.io.IOException; import java.nio.ByteBuffer; @@ -41,7 +41,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -public class VectorChunkReader implements IChunkReader { +public class AlignedChunkReader implements IChunkReader { // chunk header of the time column private final ChunkHeader timeChunkHeader; @@ -69,7 +69,7 @@ public class VectorChunkReader implements IChunkReader { * * @param filter filter */ - public VectorChunkReader(Chunk timeChunk, List<Chunk> valueChunkList, Filter filter) + public AlignedChunkReader(Chunk timeChunk, List<Chunk> valueChunkList, Filter filter) throws IOException { this.filter = filter; this.timeChunkDataBuffer = timeChunk.getData(); @@ -142,7 +142,7 @@ public class VectorChunkReader implements IChunkReader { return filter == null || filter.satisfy(pageHeader.getStatistics()); } - private VectorPageReader constructPageReaderForNextPage( + private AlignedPageReader constructPageReaderForNextPage( PageHeader timePageHeader, List<PageHeader> valuePageHeader) throws IOException { PageInfo timePageInfo = new PageInfo(); getPageInfo(timePageHeader, timeChunkDataBuffer, timeChunkHeader, timePageInfo); @@ -175,8 +175,8 @@ public class VectorChunkReader implements IChunkReader { valueDecoderList.add(null); } } - VectorPageReader vectorPageReader = - new VectorPageReader( + AlignedPageReader alignedPageReader = + new AlignedPageReader( timePageHeader, timePageInfo.pageData, timeDecoder, @@ -185,8 +185,8 @@ public class VectorChunkReader implements IChunkReader { valueDataTypeList, valueDecoderList, filter); - vectorPageReader.setDeleteIntervalList(valueDeleteIntervalList); - return vectorPageReader; + alignedPageReader.setDeleteIntervalList(valueDeleteIntervalList); + return alignedPageReader; } /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java similarity index 98% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java index 4c5290c..b4549f9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java @@ -34,7 +34,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -public class VectorPageReader implements IPageReader { +public class AlignedPageReader implements IPageReader { private final TimePageReader timePageReader; private final List<ValuePageReader> valuePageReaderList; @@ -42,7 +42,7 @@ public class VectorPageReader implements IPageReader { private Filter filter; private boolean isModified; - public VectorPageReader( + public AlignedPageReader( PageHeader timePageHeader, ByteBuffer timePageData, Decoder timeDecoder, diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java index e6bae26..e8a5c55 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java @@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.VectorChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -286,7 +286,7 @@ public class VectorMeasurementSchema valueChunkMetadata.add(chunkMetadata.get(i)); } chunkMetadataList.add( - new VectorChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } return chunkMetadataList; }
