This is an automated email from the ASF dual-hosted git repository. neuyilan pushed a commit to branch bugfix/rel_1.0_5447 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9a10d7e77e307e85c657a0d446a0c21ab0d51518 Author: HouliangQi <[email protected]> AuthorDate: Fri Feb 3 14:19:36 2023 +0800 fix the ConcurrentModificationException --- .../exception/CompactionExceptionHandler.java | 4 +-- .../execute/task/CrossSpaceCompactionTask.java | 6 ++-- .../execute/task/InnerSpaceCompactionTask.java | 4 +-- .../compaction/execute/utils/CompactionUtils.java | 2 +- .../compaction/schedule/CompactionScheduler.java | 8 +++--- .../iotdb/db/engine/storagegroup/DataRegion.java | 2 +- .../engine/storagegroup/HashLastFlushTimeMap.java | 2 +- .../storagegroup/IDTableLastFlushTimeMap.java | 2 +- .../db/engine/storagegroup/TsFileManager.java | 12 ++++---- .../cross/CrossSpaceCompactionExceptionTest.java | 32 +++++++++++----------- ...eCompactionWithFastPerformerValidationTest.java | 12 ++++---- .../inner/InnerSpaceCompactionExceptionTest.java | 9 +++--- .../SizeTieredCompactionSelectorTest.java | 4 +-- 13 files changed, 51 insertions(+), 48 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java index 70dcebf3f7..a510a77221 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java @@ -140,9 +140,9 @@ public class CompactionExceptionHandler { String fullStorageGroupName) throws IOException { TsFileResourceList unseqTsFileResourceList = - tsFileManager.getUnsequenceListByTimePartition(timePartition); + tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition); TsFileResourceList seqTsFileResourceList = - tsFileManager.getSequenceListByTimePartition(timePartition); + tsFileManager.getOrCreateSequenceListByTimePartition(timePartition); // delete compaction mods files CompactionUtils.deleteCompactionModsFile(sourceSeqResourceList, sourceUnseqResourceList); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java index 2fef6dd48d..a87cdd3d29 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java @@ -83,8 +83,10 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { serialId); this.selectedSequenceFiles = selectedSequenceFiles; this.selectedUnsequenceFiles = selectedUnsequenceFiles; - this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); - this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); + this.seqTsFileResourceList = + tsFileManager.getOrCreateSequenceListByTimePartition(timePartition); + this.unseqTsFileResourceList = + tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition); this.performer = performer; this.hashCode = this.toString().hashCode(); this.memoryCost = memoryCost; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java index f65c3799b7..a5c4b9fbd5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java @@ -90,9 +90,9 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { isHoldingReadLock[i] = false; } if (sequence) { - tsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); + tsFileResourceList = tsFileManager.getOrCreateSequenceListByTimePartition(timePartition); } else { - tsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); + tsFileResourceList = tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition); } this.hashCode = this.toString().hashCode(); collectSelectedFilesInfo(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java index 00bf68b85b..e60d4d7409 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java @@ -257,7 +257,7 @@ public class CompactionUtils { public static boolean validateTsFileResources( TsFileManager manager, String storageGroupName, long timePartition) { List<TsFileResource> resources = - manager.getSequenceListByTimePartition(timePartition).getArrayList(); + manager.getOrCreateSequenceListByTimePartition(timePartition).getArrayList(); resources.sort( (f1, f2) -> Long.compareUnsigned( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java index 7551d9b03d..0173423537 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/schedule/CompactionScheduler.java @@ -106,8 +106,8 @@ public class CompactionScheduler { List<List<TsFileResource>> taskList = innerSpaceCompactionSelector.selectInnerSpaceTask( sequence - ? tsFileManager.getSequenceListByTimePartition(timePartition) - : tsFileManager.getUnsequenceListByTimePartition(timePartition)); + ? tsFileManager.getOrCreateSequenceListByTimePartition(timePartition) + : tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition)); for (List<TsFileResource> task : taskList) { ICompactionPerformer performer = sequence @@ -147,8 +147,8 @@ public class CompactionScheduler { .createInstance(logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); List<CrossCompactionTaskResource> taskList = crossSpaceCompactionSelector.selectCrossSpaceTask( - tsFileManager.getSequenceListByTimePartition(timePartition), - tsFileManager.getUnsequenceListByTimePartition(timePartition)); + tsFileManager.getOrCreateSequenceListByTimePartition(timePartition), + tsFileManager.getOrCreateUnsequenceListByTimePartition(timePartition)); List<Long> memoryCost = crossSpaceCompactionSelector.getCompactionMemoryCost(); for (int i = 0, size = taskList.size(); i < size; ++i) { CompactionTaskManager.getInstance() diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index db4e9e8fcd..21b320e264 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -2334,7 +2334,7 @@ public class DataRegion implements IDataRegionForQuery { writeLock("loadNewTsFile"); try { List<TsFileResource> sequenceList = - tsFileManager.getSequenceListByTimePartition(newFilePartitionId); + tsFileManager.getOrCreateSequenceListByTimePartition(newFilePartitionId); int insertPos = findInsertionPosition(newTsFileResource, sequenceList); LoadTsFileType tsFileType = getLoadingTsFileType(insertPos, sequenceList); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java index 020077bb6b..50d524a3e5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java @@ -225,7 +225,7 @@ public class HashLastFlushTimeMap implements ILastFlushTimeMap { private long recoverFlushTime(long partitionId, String devicePath) { List<TsFileResource> tsFileResourceList = - tsFileManager.getSequenceListByTimePartition(partitionId); + tsFileManager.getOrCreateSequenceListByTimePartition(partitionId); for (int i = tsFileResourceList.size() - 1; i >= 0; i--) { if (tsFileResourceList.get(i).timeIndex.mayContainsDevice(devicePath)) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java index 6ad437fdc4..03dd87476a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java @@ -164,7 +164,7 @@ public class IDTableLastFlushTimeMap implements ILastFlushTimeMap { private long recoverFlushTime(long partitionId, String devicePath) { List<TsFileResource> tsFileResourceList = - tsFileManager.getSequenceListByTimePartition(partitionId); + tsFileManager.getOrCreateSequenceListByTimePartition(partitionId); for (int i = tsFileResourceList.size() - 1; i >= 0; i--) { if (tsFileResourceList.get(i).timeIndex.mayContainsDevice(devicePath)) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java index 96d0060c95..86fa5e742e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java @@ -85,21 +85,21 @@ public class TsFileManager { } } - public TsFileResourceList getSequenceListByTimePartition(long timePartition) { - readLock(); + public TsFileResourceList getOrCreateSequenceListByTimePartition(long timePartition) { + writeLock("getOrCreateSequenceListByTimePartition"); try { return sequenceFiles.computeIfAbsent(timePartition, l -> new TsFileResourceList()); } finally { - readUnlock(); + writeUnlock(); } } - public TsFileResourceList getUnsequenceListByTimePartition(long timePartition) { - readLock(); + public TsFileResourceList getOrCreateUnsequenceListByTimePartition(long timePartition) { + writeLock("getOrCreateUnsequenceListByTimePartition"); try { return unsequenceFiles.computeIfAbsent(timePartition, l -> new TsFileResourceList()); } finally { - readUnlock(); + writeUnlock(); } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index 2a4ac6a860..6bbaa0a49b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -145,8 +145,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { + TsFileResource.RESOURCE_SUFFIX) .exists()); } - Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); + Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size()); Assert.assertTrue(tsFileManager.isAllowCompaction()); } @@ -220,8 +220,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { + TsFileResource.RESOURCE_SUFFIX) .exists()); } - Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); + Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size()); Assert.assertTrue(tsFileManager.isAllowCompaction()); } @@ -253,13 +253,13 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { performer.perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (TsFileResource resource : seqResources) { - tsFileManager.getSequenceListByTimePartition(0).remove(resource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource); } for (TsFileResource resource : unseqResources) { - tsFileManager.getUnsequenceListByTimePartition(0).remove(resource); + tsFileManager.getOrCreateUnsequenceListByTimePartition(0).remove(resource); } for (TsFileResource resource : targetResources) { - tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(resource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(resource); } seqResources.get(0).getTsFile().delete(); compactionLogger.close(); @@ -301,8 +301,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { Assert.assertTrue( new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); } - Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size()); + Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals(0, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size()); Assert.assertTrue(tsFileManager.isAllowCompaction()); } @@ -356,13 +356,13 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { } CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); for (TsFileResource resource : seqResources) { - tsFileManager.getSequenceListByTimePartition(0).remove(resource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource); } for (TsFileResource resource : unseqResources) { - tsFileManager.getUnsequenceListByTimePartition(0).remove(resource); + tsFileManager.getOrCreateUnsequenceListByTimePartition(0).remove(resource); } for (TsFileResource resource : targetResources) { - tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(resource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(resource); } seqResources.get(0).remove(); @@ -418,8 +418,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { // compaction log file should not exist Assert.assertFalse(compactionLogFile.exists()); - Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size()); + Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals(0, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size()); Assert.assertTrue(tsFileManager.isAllowCompaction()); } @@ -541,8 +541,8 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { // compaction log file should not exist Assert.assertFalse(compactionLogFile.exists()); - Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); + Assert.assertEquals(4, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals(5, tsFileManager.getOrCreateUnsequenceListByTimePartition(0).size()); Assert.assertTrue(tsFileManager.isAllowCompaction()); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java index 2ec09f1daa..9aced0d1c5 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java @@ -2224,8 +2224,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac CrossCompactionTaskResource sourceFiles = crossSpaceCompactionSelector .selectCrossSpaceTask( - tsFileManager.getSequenceListByTimePartition(0), - tsFileManager.getUnsequenceListByTimePartition(0)) + tsFileManager.getOrCreateSequenceListByTimePartition(0), + tsFileManager.getOrCreateUnsequenceListByTimePartition(0)) .get(0); Assert.assertEquals(2, sourceFiles.getSeqFiles().size()); Assert.assertEquals(1, sourceFiles.getUnseqFiles().size()); @@ -2251,8 +2251,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac 0, crossSpaceCompactionSelector .selectCrossSpaceTask( - tsFileManager.getSequenceListByTimePartition(0), - tsFileManager.getUnsequenceListByTimePartition(0)) + tsFileManager.getOrCreateSequenceListByTimePartition(0), + tsFileManager.getOrCreateUnsequenceListByTimePartition(0)) .size()); // Target file of the first task should not be selected to participate in other inner compaction @@ -2271,8 +2271,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac // compaction task List<CrossCompactionTaskResource> pairs = crossSpaceCompactionSelector.selectCrossSpaceTask( - tsFileManager.getSequenceListByTimePartition(0), - tsFileManager.getUnsequenceListByTimePartition(0)); + tsFileManager.getOrCreateSequenceListByTimePartition(0), + tsFileManager.getOrCreateUnsequenceListByTimePartition(0)); Assert.assertEquals(1, pairs.size()); Assert.assertEquals(2, pairs.get(0).getSeqFiles().size()); Assert.assertEquals(1, pairs.get(0).getUnseqFiles().size()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java index c53bab30e7..25688361cb 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -181,9 +181,9 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact CompactionUtils.moveTargetFile( Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (TsFileResource resource : seqResources) { - tsFileManager.getSequenceListByTimePartition(0).remove(resource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).remove(resource); } - tsFileManager.getSequenceListByTimePartition(0).keepOrderInsert(targetResource); + tsFileManager.getOrCreateSequenceListByTimePartition(0).keepOrderInsert(targetResource); FileUtils.delete(seqResources.get(0).getTsFile().getPath()); seqResources.get(0).remove(); compactionLogger.close(); @@ -209,8 +209,9 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact } Assert.assertTrue(tsFileManager.isAllowCompaction()); - Assert.assertEquals(1, tsFileManager.getSequenceListByTimePartition(0).size()); - Assert.assertEquals(targetResource, tsFileManager.getSequenceListByTimePartition(0).get(0)); + Assert.assertEquals(1, tsFileManager.getOrCreateSequenceListByTimePartition(0).size()); + Assert.assertEquals( + targetResource, tsFileManager.getOrCreateSequenceListByTimePartition(0).get(0)); } /** diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java index 470dd48a94..730a4e5679 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java @@ -55,14 +55,14 @@ public class SizeTieredCompactionSelectorTest { Assert.assertEquals( 1, new SizeTieredCompactionSelector("root.test", "0", i, true, manager) - .selectInnerSpaceTask(manager.getSequenceListByTimePartition(i)) + .selectInnerSpaceTask(manager.getOrCreateSequenceListByTimePartition(i)) .size()); } Assert.assertEquals( 0, new SizeTieredCompactionSelector("root.test", "0", 9, true, manager) - .selectInnerSpaceTask(manager.getSequenceListByTimePartition(9)) + .selectInnerSpaceTask(manager.getOrCreateSequenceListByTimePartition(9)) .size()); } }
