This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch speed_up_recover in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 64b76852544b2dc8b504ae8cbef731ccad272a48 Author: HTHou <[email protected]> AuthorDate: Wed Jul 31 12:03:17 2024 +0800 fix package structure --- .../db/storageengine/dataregion/DataRegion.java | 4 ++-- .../dataregion/memtable/TsFileProcessor.java | 2 +- .../dataregion/tsfile/TsFileManager.java | 4 ++++ .../dataregion/tsfile/TsFileResource.java | 19 +++++---------- .../timeindex/FileTimeIndexCacheRecorder.java | 4 ++-- .../FileTimeIndexCacheReader.java | 28 +++++++++++----------- .../FileTimeIndexCacheWriter.java | 8 ++++++- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 082cde7cfa3..7c8012be62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -98,6 +98,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameG import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.VersionController; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder; +import org.apache.iotdb.db.storageengine.dataregion.utils.fileTimeIndexCache.FileTimeIndexCacheReader; import org.apache.iotdb.db.storageengine.dataregion.utils.validate.TsFileValidator; import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager; import org.apache.iotdb.db.storageengine.dataregion.wal.node.IWALNode; @@ -115,7 +116,6 @@ import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; import org.apache.iotdb.db.utils.DateTimeUtils; -import org.apache.iotdb.db.utils.writelog.FileTimeIndexCacheReader; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -961,7 +961,7 @@ public class DataRegion implements IDataRegionForQuery { boolean isSeq) { for (TsFileResource tsFileResource : resourceList) { recoverSealedTsFiles(tsFileResource, context, isSeq); - FileTimeIndexCacheRecorder.getInstance().submitTask(dataRegionSysDir, tsFileResource); + FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(dataRegionSysDir, tsFileResource); } if (config.isEnableSeparateData()) { if (!lastFlushTimeMap.checkAndCreateFlushedTimePartition(partitionId, true)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 8e599c3e35a..4d713920a57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1583,7 +1583,7 @@ public class TsFileProcessor { writer.endFile(); tsFileResource.serialize(); FileTimeIndexCacheRecorder.getInstance() - .submitTask(dataRegionInfo.getDataRegion().getDataRegionSysDir(), tsFileResource); + .logFileTimeIndex(dataRegionInfo.getDataRegion().getDataRegionSysDir(), tsFileResource); if (logger.isDebugEnabled()) { logger.debug("Ended file {}", tsFileResource); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index dc242858bce..f8f5dfaf9fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -20,10 +20,12 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder; import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager; import org.apache.tsfile.read.filter.basic.Filter; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; @@ -260,6 +262,8 @@ public class TsFileManager { .computeIfAbsent(timePartition, t -> new TsFileResourceList()) .keepOrderInsert(resource); } + FileTimeIndexCacheRecorder.getInstance() + .logFileTimeIndex(new File(storageGroupDir), resource); } } } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 0727d7e770f..a4de311d31a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -22,7 +22,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; -import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -50,7 +49,6 @@ import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.FilePathUtils; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -58,7 +56,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedOutputStream; -import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -297,16 +294,12 @@ public class TsFileResource { } public ByteBuffer serializeFileTimeIndexToByteBuffer() { - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - outputStream.writeLong(tsFileID.fileVersion); - outputStream.writeLong(tsFileID.compactionVersion); - outputStream.writeLong(timeIndex.getMinStartTime()); - outputStream.writeLong(timeIndex.getMaxEndTime()); - return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); - } catch (IOException e) { - throw new SerializationRunTimeException(e); - } + ByteBuffer buffer = ByteBuffer.allocate(4 * Long.BYTES); + buffer.putLong(tsFileID.fileVersion); + buffer.putLong(tsFileID.compactionVersion); + buffer.putLong(timeIndex.getMinStartTime()); + buffer.putLong(timeIndex.getMaxEndTime()); + return buffer; } public void updateStartTime(IDeviceID device, long time) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndexCacheRecorder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndexCacheRecorder.java index 6406cef9eaa..b69fe0a5306 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndexCacheRecorder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndexCacheRecorder.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.utils.writelog.FileTimeIndexCacheWriter; +import org.apache.iotdb.db.storageengine.dataregion.utils.fileTimeIndexCache.FileTimeIndexCacheWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +68,7 @@ public class FileTimeIndexCacheRecorder { } } - public void submitTask(File dataRegionSysDir, TsFileResource tsFileResource) { + public void logFileTimeIndex(File dataRegionSysDir, TsFileResource tsFileResource) { TsFileID tsFileID = tsFileResource.getTsFileID(); int dataRegionId = tsFileID.regionId; long partitionId = tsFileID.timePartitionId; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheReader.java similarity index 71% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheReader.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheReader.java index 2caf1f800a4..f749416234f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheReader.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.utils.writelog; +package org.apache.iotdb.db.storageengine.dataregion.utils.fileTimeIndexCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; @@ -45,19 +45,19 @@ public class FileTimeIndexCacheReader { } public void read(Map<TsFileID, FileTimeIndex> fileTimeIndexMap) throws IOException { - DataInputStream logStream = - new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); - long readLength = 0L; - while (readLength < fileLength) { - long fileVersion = logStream.readLong(); - long compactionVersion = logStream.readLong(); - long minStartTime = logStream.readLong(); - long maxEndTime = logStream.readLong(); - TsFileID tsFileID = new TsFileID(dataRegionId, partitionId, fileVersion, compactionVersion); - FileTimeIndex fileTimeIndex = new FileTimeIndex(minStartTime, maxEndTime); - fileTimeIndexMap.put(tsFileID, fileTimeIndex); - readLength += 4 * Long.BYTES; + try (DataInputStream logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath())))) { + long readLength = 0L; + while (readLength < fileLength) { + long fileVersion = logStream.readLong(); + long compactionVersion = logStream.readLong(); + long minStartTime = logStream.readLong(); + long maxEndTime = logStream.readLong(); + TsFileID tsFileID = new TsFileID(dataRegionId, partitionId, fileVersion, compactionVersion); + FileTimeIndex fileTimeIndex = new FileTimeIndex(minStartTime, maxEndTime); + fileTimeIndexMap.put(tsFileID, fileTimeIndex); + readLength += 4 * Long.BYTES; + } } - logStream.close(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java similarity index 93% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheWriter.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java index fc1fc84edd6..f1449388c35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/FileTimeIndexCacheWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.utils.writelog; +package org.apache.iotdb.db.storageengine.dataregion.utils.fileTimeIndexCache; + +import org.apache.iotdb.db.utils.writelog.ILogWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,4 +85,8 @@ public class FileTimeIndexCacheWriter implements ILogWriter { public String toString() { return "LogWriter{" + "logFile=" + logFile + '}'; } + + public File getLogFile() { + return logFile; + } }
