This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch add_meta_marker_offset in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 839dd1b39ad10f9336beb372d24ca533aea5b824 Author: qiaojialin <[email protected]> AuthorDate: Tue Apr 14 16:04:58 2020 +0800 add metaOffset in TsFileMetadata --- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 20 ++++++++++++++++++-- .../tsfile/write/writer/ForceAppendTsFileWriter.java | 20 +++----------------- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 +++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java index feb147e..3195c08 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java @@ -54,6 +54,8 @@ public class TsFileMetadata { // offset -> version private List<Pair<Long, Long>> versionInfo; + // offset of MetaMarker.SEPARATOR + private long metaOffset; /** * deserialize data from the buffer. @@ -90,6 +92,9 @@ public class TsFileMetadata { } fileMetaData.setVersionInfo(versionInfo); + // metaOffset + long metaOffset = ReadWriteIOUtils.readLong(buffer); + fileMetaData.setMetaOffset(metaOffset); // read bloom filter if (buffer.hasRemaining()) { @@ -134,10 +139,13 @@ public class TsFileMetadata { // versionInfo byteLen += ReadWriteIOUtils.write(versionInfo.size(), outputStream); for (Pair<Long, Long> versionPair : versionInfo) { - byteLen +=ReadWriteIOUtils.write(versionPair.left, outputStream); - byteLen +=ReadWriteIOUtils.write(versionPair.right, outputStream); + byteLen += ReadWriteIOUtils.write(versionPair.left, outputStream); + byteLen += ReadWriteIOUtils.write(versionPair.right, outputStream); } + // metaOffset + byteLen += ReadWriteIOUtils.write(metaOffset, outputStream); + return byteLen; } @@ -192,6 +200,14 @@ public class TsFileMetadata { this.invalidChunkNum = invalidChunkNum; } + public long getMetaOffset() { + return metaOffset; + } + + public void setMetaOffset(long metaOffset) { + this.metaOffset = metaOffset; + } + public Map<String, Pair<Long, Integer>> getDeviceMetadataIndex() { return deviceMetadataIndex; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java index a8627d6..f23e99b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java @@ -20,13 +20,9 @@ package org.apache.iotdb.tsfile.write.writer; import java.io.File; import java.io.IOException; -import java.util.Map; import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException; -import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.utils.Pair; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,11 +34,10 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter { private long truncatePosition; private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class); - private static final Logger resourceLogger = LoggerFactory.getLogger("FileMonitor"); public ForceAppendTsFileWriter(File file) throws IOException { - if (resourceLogger.isInfoEnabled()) { - resourceLogger.info("{} writer is opened.", file.getName()); + if (logger.isDebugEnabled()) { + logger.debug("{} writer is opened.", file.getName()); } this.out = new LocalTsFileOutput(file, true); this.file = file; @@ -60,17 +55,8 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter { "File " + file.getPath() + " is not a complete TsFile"); } TsFileMetadata tsFileMetadata = reader.readFileMetadata(); - Map<String, Pair<Long, Integer>> deviceMap = tsFileMetadata.getDeviceMetadataIndex(); - long firstChunkMetaPos = Long.MAX_VALUE; - for (String deviceId : deviceMap.keySet()) { - Map<String, TimeseriesMetadata> deviceMetadata = reader.readDeviceMetadata(deviceId); - for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.values()) { - firstChunkMetaPos = firstChunkMetaPos > timeseriesMetadata.getOffsetOfChunkMetaDataList() ? - timeseriesMetadata.getOffsetOfChunkMetaDataList() : firstChunkMetaPos; - } - } // truncate metadata and marker - truncatePosition = firstChunkMetaPos - 1; + truncatePosition = tsFileMetadata.getMetaOffset(); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index d26b1b9..06460b0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -218,6 +218,8 @@ public class TsFileIOWriter { */ public void endFile() throws IOException { + long metaOffset = out.getPosition(); + // serialize the SEPARATOR of MetaData ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); @@ -237,6 +239,7 @@ public class TsFileIOWriter { tsFileMetaData.setVersionInfo(versionInfo); tsFileMetaData.setTotalChunkNum(totalChunkNum); tsFileMetaData.setInvalidChunkNum(invalidChunkNum); + tsFileMetaData.setMetaOffset(metaOffset); long footerIndex = out.getPosition(); if (logger.isDebugEnabled()) {
