This is an automated email from the ASF dual-hosted git repository. tanxinyu pushed a commit to branch rel_0.12_debug_compaction_stop in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 482d8fd5509aa21c3249f1f5a99327892fc6d25d Author: Liu Xuxin <[email protected]> AuthorDate: Tue Nov 9 15:22:14 2021 +0800 add log when getting the log --- .../db/engine/compaction/TsFileManagement.java | 4 +- .../level/LevelCompactionTsFileManagement.java | 2 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 4 +- .../engine/storagegroup/StorageGroupProcessor.java | 10 ++-- .../db/engine/storagegroup/TsFileResource.java | 54 +++++++++++++++++----- .../iotdb/db/engine/upgrade/UpgradeTask.java | 8 ++-- .../iotdb/db/query/control/FileReaderManager.java | 4 +- .../org/apache/iotdb/db/utils/UpgradeUtils.java | 8 ++-- 8 files changed, 62 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java index a9d3a41..a9e09c4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java @@ -322,7 +322,7 @@ public abstract class TsFileManagement { boolean fileLockGot; boolean compactionLockGot; while (true) { - fileLockGot = seqFile.tryWriteLock(); + fileLockGot = seqFile.tryWriteLock("doubleWriteLock"); compactionLockGot = tryWriteLock(); if (fileLockGot && compactionLockGot) { @@ -359,7 +359,7 @@ public abstract class TsFileManagement { } for (TsFileResource unseqFile : unseqFiles) { - unseqFile.writeLock(); + unseqFile.writeLock("removeUnseqFiles"); try { unseqFile.remove(); } finally { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java index 0698d7e..894421c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java @@ -151,7 +151,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { } private void deleteLevelFile(TsFileResource seqFile) { - seqFile.writeLock(); + seqFile.writeLock("deleteLevelFile"); try { ChunkCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java index 36d5b5c..588c8b1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java @@ -162,7 +162,7 @@ public class MergeFileTask { return; } - seqFile.writeLock(); + seqFile.writeLock("moveMergedToOld"); try { if (Thread.currentThread().isInterrupted()) { return; @@ -348,7 +348,7 @@ public class MergeFileTask { updatePlanIndexes(seqFile); - seqFile.writeLock(); + seqFile.writeLock("moveUnmergedToNew"); try { if (Thread.currentThread().isInterrupted()) { return; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index 8361382..767c1fc 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -1395,7 +1395,7 @@ public class StorageGroupProcessor { resource.setDeleted(true); // ensure that the file is not used by any queries - if (resource.tryWriteLock()) { + if (resource.tryWriteLock("checkFileTTL")) { try { // physical removal resource.remove(); @@ -2107,7 +2107,7 @@ public class StorageGroupProcessor { return; } for (TsFileResource resource : resources) { - resource.writeLock(); + resource.writeLock("loadUpgradedResources"); try { UpgradeUtils.moveUpgradedFiles(resource); tsFileManagement.addAll(resource.getUpgradedResources(), isseq); @@ -2429,7 +2429,7 @@ public class StorageGroupProcessor { TsFileResource existingTsFile = iterator.next(); if (newTsFile.isPlanRangeCovers(existingTsFile) && !newTsFile.getTsFile().equals(existingTsFile.getTsFile()) - && existingTsFile.tryWriteLock()) { + && existingTsFile.tryWriteLock("removeFullyOverlapFiles")) { logger.info( "{} is covered by {}: [{}, {}], [{}, {}], remove it", existingTsFile, @@ -2739,7 +2739,7 @@ public class StorageGroupProcessor { if (tsFileResourceToBeDeleted == null) { return false; } - tsFileResourceToBeDeleted.writeLock(); + tsFileResourceToBeDeleted.writeLock("deleteTsfile"); try { tsFileResourceToBeDeleted.remove(); logger.info("Delete tsfile {} successfully.", tsFileResourceToBeDeleted.getTsFile()); @@ -2793,7 +2793,7 @@ public class StorageGroupProcessor { if (tsFileResourceToBeMoved == null) { return false; } - tsFileResourceToBeMoved.writeLock(); + tsFileResourceToBeMoved.writeLock("moveTsfile"); try { tsFileResourceToBeMoved.moveTo(targetDir); logger.info( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index 08d30e8..2a8dd63 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -31,7 +31,8 @@ import org.apache.iotdb.db.exception.PartitionViolationException; import org.apache.iotdb.db.service.UpgradeSevice; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; -import org.apache.iotdb.tsfile.file.metadata.*; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; @@ -50,9 +51,21 @@ import java.io.OutputStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.*; - -import static org.apache.iotdb.db.conf.IoTDBConstant.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; + +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX; +import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX; import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; @SuppressWarnings("java:S1135") // ignore todos @@ -67,6 +80,9 @@ public class TsFileResource { // tsfile private File file; + private String writeLockHolder; + private List<String> readLockHolders = new ArrayList<>(); + public static final String RESOURCE_SUFFIX = ".resource"; static final String TEMP_SUFFIX = ".temp"; @@ -403,12 +419,15 @@ public class TsFileResource { return processor; } - public void writeLock() { + public void writeLock(String holder) { if (originTsFileResource == null) { tsFileLock.writeLock(); } else { - originTsFileResource.writeLock(); + originTsFileResource.writeLock(holder); } + this.writeLockHolder = holder; + logger.warn("write lock stack of {}", holder, new RuntimeException("writeLock")); + logger.info("{} get the write lock of {}", holder, this.file); } public void writeUnlock() { @@ -417,30 +436,41 @@ public class TsFileResource { } else { originTsFileResource.writeUnlock(); } + writeLockHolder = ""; } /** * If originTsFileResource is not null, we should acquire the read lock of originTsFileResource * before construct the current TsFileResource */ - public void readLock() { + public void readLock(String holder) { if (originTsFileResource == null) { tsFileLock.readLock(); } else { - originTsFileResource.readLock(); + originTsFileResource.readLock(holder); } + readLockHolders.add(holder); + logger.warn("read lock stack of {}", holder, new RuntimeException("readLock")); + logger.info("{} get the read lock of {}", holder, this.file); } - public void readUnlock() { + public void readUnlock(String holder) { if (originTsFileResource == null) { tsFileLock.readUnlock(); } else { - originTsFileResource.readUnlock(); + originTsFileResource.readUnlock(holder); } + readLockHolders.remove(holder); } - public boolean tryWriteLock() { - return tsFileLock.tryWriteLock(); + public boolean tryWriteLock(String holder) { + if (tsFileLock.tryWriteLock()) { + this.writeLockHolder = holder; + logger.info("{} get the write lock of {}", holder, this.file); + logger.warn("write lock stack of {}", holder, new RuntimeException("writeLock")); + return true; + } + return false; } void doUpgrade() { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java index bcf29d8..120f7dd 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java @@ -82,7 +82,7 @@ public class UpgradeTask extends WrappedRunnable { } private List<TsFileResource> generateUpgradedFiles() throws IOException, WriteProcessException { - upgradeResource.readLock(); + upgradeResource.readLock("generateUpgradedFiles"); String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); List<TsFileResource> upgradedResources = new ArrayList<>(); UpgradeLog.writeUpgradeLogFile( @@ -92,13 +92,13 @@ public class UpgradeTask extends WrappedRunnable { UpgradeLog.writeUpgradeLogFile( oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE); } finally { - upgradeResource.readUnlock(); + upgradeResource.readUnlock("generateUpgradedFiles"); } return upgradedResources; } private List<TsFileResource> findUpgradedFiles() throws IOException { - upgradeResource.readLock(); + upgradeResource.readLock("findUpgradedFiles"); List<TsFileResource> upgradedResources = new ArrayList<>(); String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); UpgradeLog.writeUpgradeLogFile( @@ -122,7 +122,7 @@ public class UpgradeTask extends WrappedRunnable { UpgradeLog.writeUpgradeLogFile( oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE); } finally { - upgradeResource.readUnlock(); + upgradeResource.readUnlock("findUpgradedFiles"); } return upgradedResources; } diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java index cf329af..3fc199f 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java @@ -192,7 +192,7 @@ public class FileReaderManager implements IService { * of a reader equals zero, the reader can be closed and removed. */ void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) { - tsFile.readLock(); + tsFile.readLock("fileReaderReference"); synchronized (this) { if (!isClosed) { unclosedReferenceMap @@ -218,7 +218,7 @@ public class FileReaderManager implements IService { closedReferenceMap.get(tsFile.getTsFilePath()).decrementAndGet(); } } - tsFile.readUnlock(); + tsFile.readUnlock("fileReaderReference"); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java index f8e2b82..302a1b9 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java @@ -62,16 +62,16 @@ public class UpgradeUtils { /** judge whether a tsfile needs to be upgraded */ public static boolean isNeedUpgrade(TsFileResource tsFileResource) { - tsFileResource.readLock(); + tsFileResource.readLock("isNeedUpgrade"); // case the TsFile's length is equal to 0, the TsFile does not need to be upgraded try { if (tsFileResource.getTsFile().length() == 0) { return false; } } finally { - tsFileResource.readUnlock(); + tsFileResource.readUnlock("isNeedUpgrade"); } - tsFileResource.readLock(); + tsFileResource.readLock("isNeedUpgrade"); try (TsFileSequenceReaderForV2 tsFileSequenceReader = new TsFileSequenceReaderForV2(tsFileResource.getTsFile().getAbsolutePath())) { String versionNumber = tsFileSequenceReader.readVersionNumberV2(); @@ -85,7 +85,7 @@ public class UpgradeUtils { tsFileResource.getTsFile().getAbsolutePath(), e); } finally { - tsFileResource.readUnlock(); + tsFileResource.readUnlock("isNeedUpgrade"); } return false; }
