This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch NewTsFileV2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 140ddec0e738ee4ed85ebfb999444639f8d329ba Author: JackieTien97 <[email protected]> AuthorDate: Tue Jan 19 13:54:10 2021 +0800 some changes --- .../db/engine/cache/TimeSeriesMetadataCache.java | 2 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 7 ++--- .../tsfile/file/metadata/TimeseriesMetadata.java | 32 ++++++++++++++++++++-- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 8 +++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index 384a496..a0cf793 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -193,7 +193,7 @@ public class TimeSeriesMetadataCache { "Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device, key.measurement, key.filePath, timeseriesMetadata); } - return new TimeseriesMetadata(timeseriesMetadata); + return timeseriesMetadata; } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index 7144419..2b55d3b 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.query.reader.chunk.metadata; import java.io.IOException; import java.util.List; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.metadata.PartialPath; @@ -49,10 +48,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { } @Override - public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) - throws IOException { - List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache - .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata); + public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) { + List<ChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index 4f93f17..a8c30dd 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -22,11 +22,13 @@ package org.apache.iotdb.tsfile.file.metadata; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import org.apache.iotdb.tsfile.common.cache.Accountable; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; +import org.apache.iotdb.tsfile.utils.PublicBAOS; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -56,18 +58,25 @@ public class TimeseriesMetadata implements Accountable { // used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata private boolean isSeq = true; + private ByteBuffer chunkMetadataListByteBuffer; + + private PublicBAOS chunkMetadataListBuffer; + + private ArrayList<ChunkMetadata> chunkMetadataList; + public TimeseriesMetadata() { } public TimeseriesMetadata(byte timeSeriesMetadataType, long startOffsetOfChunkMetaDataList, int chunkMetaDataListDataSize, String measurementId, TSDataType dataType, - Statistics statistics) { + Statistics statistics, PublicBAOS chunkMetadataListBuffer) { this.timeSeriesMetadataType = timeSeriesMetadataType; this.startOffsetOfChunkMetaDataList = startOffsetOfChunkMetaDataList; this.chunkMetaDataListDataSize = chunkMetaDataListDataSize; this.measurementId = measurementId; this.dataType = dataType; this.statistics = statistics; + this.chunkMetadataListBuffer = chunkMetadataListBuffer; } public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) { @@ -86,9 +95,13 @@ public class TimeseriesMetadata implements Accountable { timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer)); timeseriesMetaData.setTSDataType(ReadWriteIOUtils.readDataType(buffer)); timeseriesMetaData.setOffsetOfChunkMetaDataList(ReadWriteIOUtils.readLong(buffer)); + int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); timeseriesMetaData - .setDataSizeOfChunkMetaDataList(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); + .setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); timeseriesMetaData.setStatistics(Statistics.deserialize(buffer, timeseriesMetaData.dataType)); + timeseriesMetaData.chunkMetadataListByteBuffer = buffer.slice(); + timeseriesMetaData.chunkMetadataListByteBuffer.limit(chunkMetaDataListDataSize); + buffer.position(buffer.position() + chunkMetaDataListDataSize); return timeseriesMetaData; } @@ -108,6 +121,8 @@ public class TimeseriesMetadata implements Accountable { byteLen += ReadWriteForEncodingUtils .writeUnsignedVarInt(chunkMetaDataListDataSize, outputStream); byteLen += statistics.serialize(outputStream); + chunkMetadataListBuffer.writeTo(outputStream); + byteLen += chunkMetadataListBuffer.size(); return byteLen; } @@ -167,6 +182,19 @@ public class TimeseriesMetadata implements Accountable { return chunkMetadataLoader.loadChunkMetadataList(this); } + public List<ChunkMetadata> getChunkMetadataList() { + if (chunkMetadataList == null) { + chunkMetadataList = new ArrayList<>(); + while (chunkMetadataListByteBuffer.hasRemaining()) { + chunkMetadataList.add(ChunkMetadata.deserializeFrom(chunkMetadataListByteBuffer, this)); + } + // minimize the storage of an ArrayList instance. + chunkMetadataList.trimToSize(); + chunkMetadataListByteBuffer = null; + } + return chunkMetadataList; + } + public boolean isModified() { return modified; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index d3290a9..f7e7d72 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -282,8 +282,8 @@ public class TsFileIOWriter { String device = path.getDevice(); // create TimeseriesMetaData + PublicBAOS publicBAOS = new PublicBAOS(); TSDataType dataType = entry.getValue().get(entry.getValue().size() - 1).getDataType(); - long offsetOfChunkMetadataList = out.getPosition(); Statistics seriesStatistics = Statistics.getStatsByType(dataType); int chunkMetadataListLength = 0; @@ -294,12 +294,12 @@ public class TsFileIOWriter { continue; } chunkMetadataListLength += chunkMetadata - .serializeTo(out.wrapAsStream(), serializeStatistic); + .serializeTo(publicBAOS, serializeStatistic); seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); } TimeseriesMetadata timeseriesMetadata = new TimeseriesMetadata( - serializeStatistic ? (byte) 1 : (byte) 0, offsetOfChunkMetadataList, - chunkMetadataListLength, path.getMeasurement(), dataType, seriesStatistics); + serializeStatistic ? (byte) 1 : (byte) 0, 0, + chunkMetadataListLength, path.getMeasurement(), dataType, seriesStatistics, publicBAOS); deviceTimeseriesMetadataMap.computeIfAbsent(device, k -> new ArrayList<>()) .add(timeseriesMetadata); }
