This is an automated email from the ASF dual-hosted git repository. JackieTien97 pushed a commit to branch rc/2.0.11 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ddf12bd5602fc6ebb9f7dab9f1206cc56a10e7bf Author: Jiang Tian <[email protected]> AuthorDate: Thu Jul 16 09:28:56 2026 +0800 Fix duplicate TsFile metric updates (#18219) * refactor addFile * record whether a file is recorded by metric * remove redundant arg --- .../iotdb/db/service/metrics/FileMetrics.java | 8 +-- .../db/service/metrics/file/TsFileMetrics.java | 19 +++++-- .../db/storageengine/dataregion/DataRegion.java | 61 ++++---------------- .../execute/task/CrossSpaceCompactionTask.java | 4 +- .../execute/task/InnerSpaceCompactionTask.java | 5 +- .../task/InsertionCrossSpaceCompactionTask.java | 12 +--- .../execute/task/SettleCompactionTask.java | 3 +- .../compaction/execute/utils/CompactionUtils.java | 17 ++---- .../dataregion/tsfile/TsFileResource.java | 17 ++++++ .../cross/InsertionCrossSpaceCompactionTest.java | 16 +----- .../utils/CompactionUpdateFileCountTest.java | 66 ++++++++++++++++++++-- .../compaction/utils/CompactionUtilsTest.java | 2 +- 12 files changed, 122 insertions(+), 108 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java index 67cc75be749..a714adfe315 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java @@ -64,12 +64,12 @@ public class FileMetrics implements IMetricSet { } // region TsFile Related Metrics Update - public void addTsFile(String database, String regionId, long size, boolean seq, String name) { - TS_FILE_METRICS.addTsFile(database, regionId, size, seq, name); + public void addTsFile(TsFileResource tsFileResource) { + TS_FILE_METRICS.addTsFile(tsFileResource); } - public void deleteTsFile(boolean seq, List<TsFileResource> tsFileResourceList) { - TS_FILE_METRICS.deleteFile(seq, tsFileResourceList); + public void deleteTsFile(List<TsFileResource> tsFileResourceList) { + TS_FILE_METRICS.deleteFile(tsFileResourceList); } public void deleteRegion(String database, String regionId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/TsFileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/TsFileMetrics.java index 2d78e2f6baf..de0c0fd23d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/TsFileMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/TsFileMetrics.java @@ -92,10 +92,17 @@ public class TsFileMetrics implements IMetricSet { } // region external update tsfile related metrics - public void addTsFile(String database, String regionId, long size, boolean seq, String name) { - updateGlobalTsFileCountAndSize(database, regionId, 1, size, seq); + public void addTsFile(TsFileResource tsFileResource) { + if (!tsFileResource.markAsRecordedByMetric()) { + return; + } + long size = tsFileResource.getTsFileSize(); + boolean seq = tsFileResource.isSeq(); + updateGlobalTsFileCountAndSize( + tsFileResource.getDatabaseName(), tsFileResource.getDataRegionId(), 1, size, seq); try { - TsFileNameGenerator.TsFileName tsFileName = TsFileNameGenerator.getTsFileName(name); + TsFileNameGenerator.TsFileName tsFileName = + TsFileNameGenerator.getTsFileName(tsFileResource.getTsFile().getName()); int level = tsFileName.getInnerCompactionCnt(); updateLevelTsFileCountAndSize(size, 1, seq, level); } catch (IOException e) { @@ -103,10 +110,14 @@ public class TsFileMetrics implements IMetricSet { } } - public void deleteFile(boolean seq, List<TsFileResource> tsFileResourceList) { + public void deleteFile(List<TsFileResource> tsFileResourceList) { for (TsFileResource tsFileResource : tsFileResourceList) { + if (!tsFileResource.markAsUnrecordedByMetric()) { + continue; + } String name = tsFileResource.getTsFile().getName(); long size = tsFileResource.getTsFileSize(); + boolean seq = tsFileResource.isSeq(); updateGlobalTsFileCountAndSize( tsFileResource.getDatabaseName(), tsFileResource.getDataRegionId(), -1, -size, seq); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 63a6269b190..7f29ab552df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -628,13 +628,7 @@ public class DataRegion implements IDataRegionForQuery { // tsFiles without resource file are unsealed for (TsFileResource resource : value) { if (resource.resourceFileExists()) { - FileMetrics.getInstance() - .addTsFile( - resource.getDatabaseName(), - resource.getDataRegionId(), - resource.getTsFile().length(), - true, - resource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(resource); if (ModificationFile.getExclusiveMods(resource.getTsFile()).exists()) { // update mods file metrics resource.getExclusiveModFile(); @@ -662,13 +656,7 @@ public class DataRegion implements IDataRegionForQuery { // tsFiles without resource file are unsealed for (TsFileResource resource : unseqTsFiles) { if (resource.resourceFileExists()) { - FileMetrics.getInstance() - .addTsFile( - resource.getDatabaseName(), - resource.getDataRegionId(), - resource.getTsFile().length(), - false, - resource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(resource); } else { WALRecoverListener recoverListener = recoverUnsealedTsFile(resource, dataRegionRecoveryContext, false); @@ -975,13 +963,7 @@ public class DataRegion implements IDataRegionForQuery { } updateDeviceLastFlushTime(tsFileResource); tsFileResourceManager.registerSealedTsFileResource(tsFileResource); - FileMetrics.getInstance() - .addTsFile( - tsFileResource.getDatabaseName(), - tsFileResource.getDataRegionId(), - tsFileResource.getTsFile().length(), - recoverPerformer.isSequence(), - tsFileResource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(tsFileResource); } else { // the last file is not closed, continue writing to it RestorableTsFileIOWriter writer = recoverPerformer.getWriter(); @@ -2302,7 +2284,7 @@ public class DataRegion implements IDataRegionForQuery { tsFileResourceList.addAll(tsFileManager.getTsFileList(false)); tsFileResourceList.forEach( x -> { - FileMetrics.getInstance().deleteTsFile(x.isSeq(), Collections.singletonList(x)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(x)); try { x.removeModFile(); } catch (IOException e) { @@ -2578,13 +2560,7 @@ public class DataRegion implements IDataRegionForQuery { closedTsFileResources.add(tsFileProcessor.getTsFileResource()); } for (TsFileResource resource : closedTsFileResources) { - FileMetrics.getInstance() - .addTsFile( - resource.getDatabaseName(), - resource.getDataRegionId(), - resource.getTsFileSize(), - resource.isSeq(), - resource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(resource); } WritingMetrics.getInstance().recordActiveTimePartitionCount(-1); logger.info(StorageEngineMessages.FILES_WERE_CLOSED, closedTsFileResources.size()); @@ -3708,8 +3684,7 @@ public class DataRegion implements IDataRegionForQuery { tsFileManager.remove(tsFileResource, tsFileResource.isSeq()); tsFileResource.writeLock(); try { - FileMetrics.getInstance() - .deleteTsFile(tsFileResource.isSeq(), Collections.singletonList(tsFileResource)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(tsFileResource)); tsFileResource.remove(); logger.info( StorageEngineMessages.REMOVE_TSFILE_DIRECTLY_WHEN_DELETE_DATA, @@ -3752,8 +3727,7 @@ public class DataRegion implements IDataRegionForQuery { tsFileManager.remove(tsFileResource, tsFileResource.isSeq()); tsFileResource.writeLock(); try { - FileMetrics.getInstance() - .deleteTsFile(tsFileResource.isSeq(), Collections.singletonList(tsFileResource)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(tsFileResource)); tsFileResource.remove(); logger.info( StorageEngineMessages.REMOVE_TSFILE_DIRECTLY_WHEN_DELETE_DATA, @@ -3855,13 +3829,7 @@ public class DataRegion implements IDataRegionForQuery { } if (!isValidateTsFileFailed) { TsFileResource tsFileResource = tsFileProcessor.getTsFileResource(); - FileMetrics.getInstance() - .addTsFile( - tsFileResource.getDatabaseName(), - tsFileResource.getDataRegionId(), - tsFileResource.getTsFileSize(), - tsFileProcessor.isSequence(), - tsFileResource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(tsFileResource); } } @@ -4226,13 +4194,7 @@ public class DataRegion implements IDataRegionForQuery { TableDiskUsageIndex.getInstance() .write(databaseName, newTsFileResource.getTsFileID(), stringLongMap)); - FileMetrics.getInstance() - .addTsFile( - newTsFileResource.getDatabaseName(), - newTsFileResource.getDataRegionId(), - newTsFileResource.getTsFile().length(), - false, - newTsFileResource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(newTsFileResource); if (config.isEnableSeparateData()) { final DataRegionId dataRegionId = @@ -4645,8 +4607,7 @@ public class DataRegion implements IDataRegionForQuery { if (sequenceResource.getTsFile().getName().equals(fileToBeUnloaded.getName())) { unloadedTsFileResource = sequenceResource; tsFileManager.remove(unloadedTsFileResource, true); - FileMetrics.getInstance() - .deleteTsFile(true, Collections.singletonList(unloadedTsFileResource)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(unloadedTsFileResource)); break; } } @@ -4658,7 +4619,7 @@ public class DataRegion implements IDataRegionForQuery { unloadedTsFileResource = unsequenceResource; tsFileManager.remove(unloadedTsFileResource, false); FileMetrics.getInstance() - .deleteTsFile(false, Collections.singletonList(unloadedTsFileResource)); + .deleteTsFile(Collections.singletonList(unloadedTsFileResource)); break; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java index 1a6703e4b67..345bca19b31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java @@ -357,8 +357,8 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { throw new CompactionRecoverException(StorageEngineMessages.SOURCE_FILES_CANNOT_BE_DELETED); } if (recoverMemoryStatus) { - FileMetrics.getInstance().deleteTsFile(true, selectedSequenceFiles); - FileMetrics.getInstance().deleteTsFile(false, selectedUnsequenceFiles); + FileMetrics.getInstance().deleteTsFile(selectedSequenceFiles); + FileMetrics.getInstance().deleteTsFile(selectedUnsequenceFiles); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index acc2ff500a9..1b0fb3205be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -435,8 +435,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { isHoldingWriteLock[i] = true; } - CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics( - filesView.sourceFilesInLog, filesView.sequence); + CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(filesView.sourceFilesInLog); updateTableSizeCache(); CompactionMetrics.getInstance().recordSummaryInfo(summary); } @@ -599,7 +598,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { throw new CompactionRecoverException(StorageEngineMessages.SOURCE_FILES_CANNOT_BE_DELETED); } if (recoverMemoryStatus) { - FileMetrics.getInstance().deleteTsFile(filesView.sequence, filesView.sourceFilesInLog); + FileMetrics.getInstance().deleteTsFile(filesView.sourceFilesInLog); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java index 6ae9791401a..60c1bdc106d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java @@ -371,18 +371,10 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask { } private void updateFileMetrics() { - // Here the target file is used for updating metrics because the source file - // has been deleted here. // The statistics of the mods file can be left unchanged, as it does not // differentiate between sequence or unsequence. - FileMetrics.getInstance().deleteTsFile(false, Collections.singletonList(targetFile)); - FileMetrics.getInstance() - .addTsFile( - targetFile.getDatabaseName(), - targetFile.getDataRegionId(), - targetFile.getTsFileSize(), - true, - targetFile.getTsFile().getName()); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(unseqFileToInsert)); + FileMetrics.getInstance().addTsFile(targetFile); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java index 388197e66e3..4de88103322 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java @@ -226,8 +226,7 @@ public class SettleCompactionTask extends InnerSpaceCompactionTask { .STORAGE_LOG_SETTLE_TASK_DELETES_FULLY_DIRTY_TSFILE_SUCCESSFULLY_18D81225, resource.getTsFile().getAbsolutePath()); if (recoverMemoryStatus) { - FileMetrics.getInstance() - .deleteTsFile(resource.isSeq(), Collections.singletonList(resource)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(resource)); } } else { LOGGER.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 5798f8fd382..d26ecdb2129 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -268,13 +268,7 @@ public class CompactionUtils { } public static void addFilesToFileMetrics(TsFileResource resource) { - FileMetrics.getInstance() - .addTsFile( - resource.getDatabaseName(), - resource.getDataRegionId(), - resource.getTsFile().length(), - resource.isSeq(), - resource.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(resource); } private static void updateOneTargetMods(TsFileResource targetFile, Set<ModEntry> modifications) @@ -396,16 +390,15 @@ public class CompactionUtils { public static void deleteSourceTsFileAndUpdateFileMetrics( List<TsFileResource> sourceSeqResourceList, List<TsFileResource> sourceUnseqResourceList) { - deleteSourceTsFileAndUpdateFileMetrics(sourceSeqResourceList, true); - deleteSourceTsFileAndUpdateFileMetrics(sourceUnseqResourceList, false); + deleteSourceTsFileAndUpdateFileMetrics(sourceSeqResourceList); + deleteSourceTsFileAndUpdateFileMetrics(sourceUnseqResourceList); } - public static void deleteSourceTsFileAndUpdateFileMetrics( - List<TsFileResource> resources, boolean seq) { + public static void deleteSourceTsFileAndUpdateFileMetrics(List<TsFileResource> resources) { for (TsFileResource resource : resources) { deleteTsFileResourceWithoutLock(resource); } - FileMetrics.getInstance().deleteTsFile(seq, resources); + FileMetrics.getInstance().deleteTsFile(resources); } public static void deleteTsFileResourceWithoutLock(TsFileResource resource) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index bffa3a0f1eb..a5ec8a555eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -92,6 +92,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -159,6 +160,8 @@ public class TsFileResource implements PersistentResource, Cloneable { private boolean isSeq; + private AtomicBoolean isRecordedByMetric = new AtomicBoolean(false); + private final FSFactory fsFactory = FSFactoryProducer.getFSFactory(); private DataRegion.SettleTsFileCallBack settleTsFileCallBack; @@ -259,6 +262,7 @@ public class TsFileResource implements PersistentResource, Cloneable { this.originTsFileResource = originTsFileResource; this.tsFileID = originTsFileResource.tsFileID; this.isSeq = originTsFileResource.isSeq; + this.isRecordedByMetric = originTsFileResource.isRecordedByMetric; this.tierLevel = originTsFileResource.tierLevel; } @@ -1372,6 +1376,18 @@ public class TsFileResource implements PersistentResource, Cloneable { return isSeq; } + public boolean isRecordedByMetric() { + return isRecordedByMetric.get(); + } + + public boolean markAsRecordedByMetric() { + return isRecordedByMetric.compareAndSet(false, true); + } + + public boolean markAsUnrecordedByMetric() { + return isRecordedByMetric.compareAndSet(true, false); + } + public int compareIndexDegradePriority(TsFileResource tsFileResource) { int cmp = timeIndex.compareDegradePriority(tsFileResource.timeIndex); return cmp == 0 ? file.getAbsolutePath().compareTo(tsFileResource.file.getAbsolutePath()) : cmp; @@ -1665,6 +1681,7 @@ public class TsFileResource implements PersistentResource, Cloneable { cloned.sharedModFileOffset = this.sharedModFileOffset; cloned.compactionModFile = this.compactionModFile; cloned.isSeq = this.isSeq; + cloned.isRecordedByMetric = this.isRecordedByMetric; cloned.tsFileRepairStatus = this.tsFileRepairStatus; cloned.settleTsFileCallBack = this.settleTsFileCallBack; cloned.deviceTimeIndexRamSize = this.deviceTimeIndexRamSize; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java index 0e0516a46c0..73c0bc2a721 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java @@ -564,13 +564,7 @@ public class InsertionCrossSpaceCompactionTest extends AbstractCompactionTest { TsFileResource unseqResource1 = generateSingleNonAlignedSeriesFileWithDevices( "2-2-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new TimeRange(1, 4)}, false); - FileMetrics.getInstance() - .addTsFile( - unseqResource1.getDatabaseName(), - unseqResource1.getDataRegionId(), - unseqResource1.getTsFileSize(), - false, - unseqResource1.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(unseqResource1); long seqFileNumBeforeCompaction = FileMetrics.getInstance().getFileCount(true); long unseqFileNumBeforeCompaction = FileMetrics.getInstance().getFileCount(false); @@ -588,13 +582,7 @@ public class InsertionCrossSpaceCompactionTest extends AbstractCompactionTest { TsFileResource unseqResource2 = generateSingleNonAlignedSeriesFileWithDevices( "3-3-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new TimeRange(5, 6)}, false); - FileMetrics.getInstance() - .addTsFile( - unseqResource2.getDatabaseName(), - unseqResource2.getDataRegionId(), - unseqResource2.getTsFileSize(), - false, - unseqResource2.getTsFile().getName()); + FileMetrics.getInstance().addTsFile(unseqResource2); seqFileNumBeforeCompaction = FileMetrics.getInstance().getFileCount(true); unseqFileNumBeforeCompaction = FileMetrics.getInstance().getFileCount(false); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUpdateFileCountTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUpdateFileCountTest.java index 1740eb2f1c1..c79cd467603 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUpdateFileCountTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUpdateFileCountTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.exception.write.WriteProcessException; import org.junit.After; @@ -35,6 +36,9 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; public class CompactionUpdateFileCountTest extends AbstractCompactionTest { @@ -53,10 +57,11 @@ public class CompactionUpdateFileCountTest extends AbstractCompactionTest { public void testSeqSpaceCompactionFileMetric() throws MetadataException, IOException, WriteProcessException { registerTimeseriesInMManger(2, 3, false); - long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); - long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); createFiles(1, 2, 3, 100, 1, 0, 50, 0, false, true); createFiles(1, 2, 3, 50, 200, 30000, 50, 50, false, true); + seqResources.forEach(FileMetrics.getInstance()::addTsFile); + long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); + long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task = new InnerSpaceCompactionTask( @@ -70,10 +75,11 @@ public class CompactionUpdateFileCountTest extends AbstractCompactionTest { public void testUnSeqSpaceCompactionFileMetric() throws MetadataException, IOException, WriteProcessException { registerTimeseriesInMManger(2, 3, false); - long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); - long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); createFiles(1, 2, 3, 100, 1, 0, 50, 0, false, false); createFiles(1, 2, 3, 50, 20, 10000, 50, 50, false, false); + unseqResources.forEach(FileMetrics.getInstance()::addTsFile); + long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); + long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); tsFileManager.addAll(unseqResources, false); InnerSpaceCompactionTask task = new InnerSpaceCompactionTask( @@ -87,10 +93,12 @@ public class CompactionUpdateFileCountTest extends AbstractCompactionTest { public void testCrossSpaceCompactionFileMetric() throws MetadataException, IOException, WriteProcessException { registerTimeseriesInMManger(2, 3, false); - long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); - long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); createFiles(1, 2, 3, 100, 1, 0, 50, 0, false, true); createFiles(3, 2, 3, 50, 20, 10000, 50, 50, false, false); + seqResources.forEach(FileMetrics.getInstance()::addTsFile); + unseqResources.forEach(FileMetrics.getInstance()::addTsFile); + long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); + long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); CrossSpaceCompactionTask task = @@ -106,4 +114,50 @@ public class CompactionUpdateFileCountTest extends AbstractCompactionTest { Assert.assertEquals(initSeqFileNum, FileMetrics.getInstance().getFileCount(true)); Assert.assertEquals(initUnSeqFileNum - 3, FileMetrics.getInstance().getFileCount(false)); } + + @Test + public void testRepeatedlyUpdateFileMetric() + throws MetadataException, IOException, WriteProcessException { + registerTimeseriesInMManger(2, 3, false); + createFiles(1, 2, 3, 100, 1, 0, 50, 0, false, true); + TsFileResource resource = seqResources.get(0); + long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); + + try { + FileMetrics.getInstance().addTsFile(resource); + FileMetrics.getInstance().addTsFile(resource); + Assert.assertTrue(resource.isRecordedByMetric()); + Assert.assertEquals(initSeqFileNum + 1, FileMetrics.getInstance().getFileCount(true)); + + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(resource)); + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(resource)); + Assert.assertFalse(resource.isRecordedByMetric()); + Assert.assertEquals(initSeqFileNum, FileMetrics.getInstance().getFileCount(true)); + } finally { + FileMetrics.getInstance().deleteTsFile(Collections.singletonList(resource)); + } + } + + @Test + public void testDeleteFileMetricByResourceSequence() + throws MetadataException, IOException, WriteProcessException { + registerTimeseriesInMManger(2, 3, false); + createFiles(1, 2, 3, 100, 1, 0, 50, 0, false, true); + createFiles(1, 2, 3, 50, 200, 30000, 50, 50, false, false); + List<TsFileResource> resources = Arrays.asList(seqResources.get(0), unseqResources.get(0)); + long initSeqFileNum = FileMetrics.getInstance().getFileCount(true); + long initUnSeqFileNum = FileMetrics.getInstance().getFileCount(false); + + try { + resources.forEach(FileMetrics.getInstance()::addTsFile); + Assert.assertEquals(initSeqFileNum + 1, FileMetrics.getInstance().getFileCount(true)); + Assert.assertEquals(initUnSeqFileNum + 1, FileMetrics.getInstance().getFileCount(false)); + + FileMetrics.getInstance().deleteTsFile(resources); + Assert.assertEquals(initSeqFileNum, FileMetrics.getInstance().getFileCount(true)); + Assert.assertEquals(initUnSeqFileNum, FileMetrics.getInstance().getFileCount(false)); + } finally { + FileMetrics.getInstance().deleteTsFile(resources); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java index 2972c0ee929..97427b16321 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java @@ -85,7 +85,7 @@ public class CompactionUtilsTest extends AbstractCompactionTest { Assert.assertEquals( modFileSizeBefore + totalModFileSize, FileMetrics.getInstance().getModFileSize()); - CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(new ArrayList<>(seqResources), true); + CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(new ArrayList<>(seqResources)); Assert.assertEquals(modFileNumBefore, FileMetrics.getInstance().getModFileNum()); Assert.assertEquals(modFileSizeBefore, FileMetrics.getInstance().getModFileSize());
