This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch fix-ttl-cannot-delete in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 67825a42dbe0af29bb3f178351cd28b761ff67cd Author: Liu Xuxin <[email protected]> AuthorDate: Wed May 17 17:54:02 2023 +0800 fix not releasing read lock when fail to allocate memory for compaction task (#9872) --- .../exception/CompactionExceptionHandler.java | 2 +- .../execute/task/CrossSpaceCompactionTask.java | 39 ++++++----- .../execute/task/InnerSpaceCompactionTask.java | 6 +- .../impl/SizeTieredCompactionSelector.java | 2 +- .../utils/CrossSpaceCompactionCandidate.java | 4 +- .../db/engine/settle/SettleRequestHandler.java | 2 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 2 +- .../db/engine/storagegroup/TsFileResource.java | 8 +-- .../engine/storagegroup/TsFileResourceStatus.java | 3 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 2 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 2 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 2 +- .../db/tools/settle/TsFileAndModSettleTool.java | 6 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +- .../org/apache/iotdb/db/utils/UpgradeUtils.java | 2 +- .../iotdb/db/engine/cache/ChunkCacheTest.java | 4 +- .../engine/compaction/AbstractCompactionTest.java | 6 +- .../db/engine/compaction/MemoryControlTest.java | 75 ++++++++++++++++++++++ .../cross/CrossSpaceCompactionExceptionTest.java | 2 +- .../CrossSpaceCompactionWithFastPerformerTest.java | 6 +- ...eCompactionWithFastPerformerValidationTest.java | 4 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 6 +- .../db/engine/compaction/cross/MergeTest.java | 6 +- .../cross/RewriteCompactionFileSelectorTest.java | 38 +++++------ .../inner/AbstractInnerSpaceCompactionTest.java | 4 +- .../inner/InnerCompactionMoreDataTest.java | 2 +- .../compaction/inner/InnerCompactionTest.java | 4 +- .../inner/sizetiered/SizeTieredCompactionTest.java | 10 +-- .../utils/MultiTsFileDeviceIteratorTest.java | 12 ++-- .../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +- .../engine/storagegroup/FakedTsFileResource.java | 2 +- .../db/engine/storagegroup/TsFileResourceTest.java | 2 +- .../execution/operator/AlignedSeriesTestUtil.java | 6 +- .../AlignedSeriesScanLimitOffsetPushDownTest.java | 10 +-- .../query/reader/series/SeriesReaderTestUtil.java | 6 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 8 +-- .../iotdb/db/rescon/ResourceManagerTest.java | 12 ++-- 37 files changed, 196 insertions(+), 115 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java index a510a77221..49a9cd0f3d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionExceptionHandler.java @@ -240,7 +240,7 @@ public class CompactionExceptionHandler { continue; } else { // set target resources to CLOSED, so that they can be selected to compact - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); } if (!TsFileUtils.isTsFileComplete(targetResource.getTsFile())) { LOGGER.error( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java index 1eddd86077..705942b12f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java @@ -98,15 +98,6 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { @Override public boolean doCompaction() { - try { - SystemInfo.getInstance().addCompactionMemoryCost(memoryCost, 60); - } catch (InterruptedException e) { - LOGGER.error("Interrupted when allocating memory for compaction", e); - return false; - } catch (CompactionMemoryNotEnoughException e) { - LOGGER.error("No enough memory for current compaction task {}", this, e); - return false; - } boolean isSuccess = true; try { if (!tsFileManager.isAllowCompaction()) { @@ -224,7 +215,7 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { TsFileMetricManager.getInstance().addFile(targetResource.getTsFileSize(), true); // set target resources to CLOSED, so that they can be selected to compact - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); } else { // target resource is empty after compaction, then delete it targetResource.remove(); @@ -295,15 +286,15 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { } private void releaseAllLock() { - selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); - selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); + selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); for (TsFileResource tsFileResource : holdReadLockList) { tsFileResource.readUnlock(); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); } for (TsFileResource tsFileResource : holdWriteLockList) { tsFileResource.writeUnlock(); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); } holdReadLockList.clear(); holdWriteLockList.clear(); @@ -353,8 +344,8 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { @Override public void resetCompactionCandidateStatusForAllSourceFiles() { - selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); - selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); + selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); } private long deleteOldFiles(List<TsFileResource> tsFileResourceList) throws IOException { @@ -380,7 +371,21 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { @Override public boolean checkValidAndSetMerging() { - return addReadLock(selectedSequenceFiles) && addReadLock(selectedUnsequenceFiles); + try { + SystemInfo.getInstance().addCompactionMemoryCost(memoryCost, 60); + } catch (InterruptedException e) { + LOGGER.error("Interrupted when allocating memory for compaction", e); + return false; + } catch (CompactionMemoryNotEnoughException e) { + LOGGER.error("No enough memory for current compaction task {}", this, e); + return false; + } + boolean addReadLockSuccess = + addReadLock(selectedSequenceFiles) && addReadLock(selectedUnsequenceFiles); + if (!addReadLockSuccess) { + SystemInfo.getInstance().resetCompactionMemoryCost(memoryCost); + } + return addReadLockSuccess; } private boolean addReadLock(List<TsFileResource> tsFileResourceList) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java index d5a99e57a6..f04c389b28 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java @@ -239,7 +239,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { .addFile(targetTsFileResource.getTsFile().length(), sequence); // set target resource to CLOSED, so that it can be selected to compact - targetTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + targetTsFileResource.setStatus(TsFileResourceStatus.NORMAL); } else { // target resource is empty after compaction, then delete it targetTsFileResource.remove(); @@ -400,7 +400,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { @Override public void resetCompactionCandidateStatusForAllSourceFiles() { - selectedTsFileResourceList.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + selectedTsFileResourceList.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); } /** @@ -418,7 +418,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { } try { if (!resource.isDeleted()) { - selectedTsFileResourceList.get(i).setStatus(TsFileResourceStatus.CLOSED); + selectedTsFileResourceList.get(i).setStatus(TsFileResourceStatus.NORMAL); } } catch (Throwable e) { LOGGER.error("Exception occurs when resetting resource status", e); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java index df88248bf2..805abe2f1d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java @@ -114,7 +114,7 @@ public class SizeTieredCompactionSelector selectedFileSize = 0L; continue; } - if (currentFile.getStatus() != TsFileResourceStatus.CLOSED) { + if (currentFile.getStatus() != TsFileResourceStatus.NORMAL) { selectedFileList.clear(); selectedFileSize = 0L; continue; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java index bedf2e2fbc..04da98652a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java @@ -142,7 +142,7 @@ public class CrossSpaceCompactionCandidate { private List<TsFileResourceCandidate> filterUnseqResource(List<TsFileResource> unseqResources) { List<TsFileResourceCandidate> ret = new ArrayList<>(); for (TsFileResource resource : unseqResources) { - if (resource.getStatus() != TsFileResourceStatus.CLOSED || !resource.getTsFile().exists()) { + if (resource.getStatus() != TsFileResourceStatus.NORMAL || !resource.getTsFile().exists()) { break; } else if (resource.stillLives(ttlLowerBound)) { ret.add(new TsFileResourceCandidate(resource)); @@ -198,7 +198,7 @@ public class CrossSpaceCompactionCandidate { // although we do the judgement here, the task should be validated before executing because // the status of file may be changed after the task is submitted to queue this.isValidCandidate = - tsFileResource.getStatus() == TsFileResourceStatus.CLOSED + tsFileResource.getStatus() == TsFileResourceStatus.NORMAL && tsFileResource.getTsFile().exists(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java b/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java index 5701820d07..fa84d422f7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java @@ -229,7 +229,7 @@ public class SettleRequestHandler { for (TsFileResource tsFileResource : allTsFileResourceList) { File tsFile = tsFileResource.getTsFile(); if (tsFileNames.contains(tsFile.getName())) { - if (tsFileResource.getStatus() != TsFileResourceStatus.CLOSED) { + if (tsFileResource.getStatus() != TsFileResourceStatus.NORMAL) { return RpcUtils.getStatus( TSStatusCode.ILLEGAL_PARAMETER, "The TsFile is not valid: " + tsFile.getAbsolutePath()); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index b82fc128ba..e609ddc4c1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -732,7 +732,7 @@ public class DataRegion implements IDataRegionForQuery { List<TsFileResource> upgradeRet = new ArrayList<>(); for (File f : upgradeFiles) { TsFileResource fileResource = new TsFileResource(f); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); // make sure the flush command is called before IoTDB is down. fileResource.deserializeFromOldFile(); upgradeRet.add(fileResource); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index f9d567196b..e91fe3ffc8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -461,7 +461,7 @@ public class TsFileResource { } public void close() throws IOException { - this.setStatus(TsFileResourceStatus.CLOSED); + this.setStatus(TsFileResourceStatus.NORMAL); closeWithoutSettingStatus(); } @@ -625,8 +625,8 @@ public class TsFileResource { public void setStatus(TsFileResourceStatus status) { switch (status) { - case CLOSED: - this.status = TsFileResourceStatus.CLOSED; + case NORMAL: + this.status = TsFileResourceStatus.NORMAL; break; case UNCLOSED: this.status = TsFileResourceStatus.UNCLOSED; @@ -645,7 +645,7 @@ public class TsFileResource { } break; case COMPACTION_CANDIDATE: - if (this.status == TsFileResourceStatus.CLOSED) { + if (this.status == TsFileResourceStatus.NORMAL) { this.status = TsFileResourceStatus.COMPACTION_CANDIDATE; } else { throw new RuntimeException( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceStatus.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceStatus.java index 42eaf481d3..4c1c95ca7e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceStatus.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceStatus.java @@ -20,7 +20,8 @@ package org.apache.iotdb.db.engine.storagegroup; public enum TsFileResourceStatus { UNCLOSED, - CLOSED, + /** The resource in status NORMAL, COMPACTION_CANDIDATE, COMPACTING, DELETED is all CLOSED. */ + NORMAL, COMPACTION_CANDIDATE, COMPACTING, DELETED diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 4b4a89ff48..a5c7bafe36 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -2272,7 +2272,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> resource.deserialize(); } - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); return resource; } } diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index cee707e0ac..e86f5685ae 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -505,7 +505,7 @@ public class TsFileSplitByPartitionTool implements AutoCloseable { } tsFileResource.setMinPlanIndex(minPlanIndex); tsFileResource.setMaxPlanIndex(maxPlanIndex); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.serialize(); return tsFileResource; } diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java index 336a5681c6..2752bdbc5f 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java @@ -279,7 +279,7 @@ public class TsFileSplitTool { } tsFileResource.setMinPlanIndex(minPlanIndex); tsFileResource.setMaxPlanIndex(maxPlanIndex); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.serialize(); return tsFileResource; diff --git a/server/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java b/server/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java index 7eb65d9d15..abaa47e185 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/settle/TsFileAndModSettleTool.java @@ -67,7 +67,7 @@ public class TsFileAndModSettleTool { for (Map.Entry<String, Integer> entry : getInstance().recoverSettleFileMap.entrySet()) { String path = entry.getKey(); TsFileResource resource = new TsFileResource(new File(path)); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); oldTsFileResources.put(resource.getTsFile().getName(), resource); } List<File> tsFiles = checkArgs(args); @@ -75,7 +75,7 @@ public class TsFileAndModSettleTool { if (!oldTsFileResources.containsKey(file.getName())) { if (new File(file + TsFileResource.RESOURCE_SUFFIX).exists()) { TsFileResource resource = new TsFileResource(file); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); oldTsFileResources.put(file.getName(), resource); } } @@ -335,7 +335,7 @@ public class TsFileAndModSettleTool { // move .resource File newTsFileResource.setFile(fsFactory.getFile(oldTsFile.getParent(), newTsFile.getName())); - newTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + newTsFileResource.setStatus(TsFileResourceStatus.NORMAL); try { newTsFileResource.serialize(); } catch (IOException e) { diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 1dc6500ac4..8cc7a6f221 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -101,7 +101,7 @@ public class FileLoaderUtils { resource.updateEndTime(device, chunkMetadata.getEndTime()); } } - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); return resource; } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java index d31cc94cbe..47e9b65e9f 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/UpgradeUtils.java @@ -119,7 +119,7 @@ public class UpgradeUtils { if (fsFactory.getFile(partitionDir, newModsFile.getName()).exists()) { upgradedResource.getModFile(); } - upgradedResource.setStatus(TsFileResourceStatus.CLOSED); + upgradedResource.setStatus(TsFileResourceStatus.NORMAL); upgradedResource.serialize(); // delete generated temp resource file Files.delete(tempResourceFile.toPath()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java index aa5f113c0b..8afb2d5207 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java @@ -150,7 +150,7 @@ public class ChunkCacheTest { Assert.assertTrue(file.getParentFile().mkdirs()); } TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(i); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); @@ -161,7 +161,7 @@ public class ChunkCacheTest { Assert.assertTrue(file.getParentFile().mkdirs()); } TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(i + seqFileNum); unseqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java index 4c5f2ae496..483670ada4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java @@ -316,7 +316,7 @@ public class AbstractCompactionTest { startTime + pointNum * i + timeInterval * i + pointNum - 1); } resource.updatePlanIndexes(fileVersion); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); resource.serialize(); if (isSeq) { seqResources.add(resource); @@ -344,7 +344,7 @@ public class AbstractCompactionTest { } resource.updatePlanIndexes(fileVersion); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); resource.serialize(); if (isSeq) { seqResources.add(resource); @@ -588,7 +588,7 @@ public class AbstractCompactionTest { } TsFileResource resource = new TsFileResource(new File(filePath)); resource.updatePlanIndexes(fileVersion++); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); return resource; } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java new file mode 100644 index 0000000000..c71356ca9a --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.compaction; + +import org.apache.iotdb.db.engine.compaction.execute.task.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.storagegroup.TsFileManager; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class MemoryControlTest { + @Test + public void testFailedToAllocateMemoryInCrossTask() throws Exception { + List<TsFileResource> sequenceFiles = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + sequenceFiles.add( + new TsFileResource( + new File(String.format("%d-%d-0-0.tsfile", i, i)), TsFileResourceStatus.NORMAL)); + } + List<TsFileResource> unsequenceFiles = new ArrayList<>(); + for (int i = 11; i <= 20; i++) { + unsequenceFiles.add( + new TsFileResource( + new File(String.format("%d-%d-0-0.tsfile", i, i)), TsFileResourceStatus.NORMAL)); + } + TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(tsFileManager.getStorageGroupName()).thenReturn("root.sg"); + Mockito.when(tsFileManager.getDataRegionId()).thenReturn("1"); + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( + 0L, + tsFileManager, + sequenceFiles, + unsequenceFiles, + null, + new AtomicInteger(0), + 1024L * 1024L * 1024L * 50L, + 0); + boolean success = task.checkValidAndSetMerging(); + Assert.assertFalse(success); + for (TsFileResource tsFileResource : sequenceFiles) { + Assert.assertEquals(TsFileResourceStatus.NORMAL, tsFileResource.getStatus()); + Assert.assertTrue(tsFileResource.tryWriteLock()); + } + for (TsFileResource tsFileResource : unsequenceFiles) { + Assert.assertEquals(TsFileResourceStatus.NORMAL, tsFileResource.getStatus()); + Assert.assertTrue(tsFileResource.tryWriteLock()); + } + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index 6bbaa0a49b..c9eeb45177 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -626,7 +626,7 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { } else { Assert.assertTrue(resource.getTsFile().exists()); Assert.assertTrue(resource.resourceFileExists()); - Assert.assertEquals(TsFileResourceStatus.CLOSED, resource.getStatus()); + Assert.assertEquals(TsFileResourceStatus.NORMAL, resource.getStatus()); } } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java index 2a9d89cb47..2f7b7ed4d3 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java @@ -437,7 +437,7 @@ public class CrossSpaceCompactionWithFastPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); @@ -741,7 +741,7 @@ public class CrossSpaceCompactionWithFastPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); @@ -1044,7 +1044,7 @@ public class CrossSpaceCompactionWithFastPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java index 2f7c1bdad2..fd470fb0e1 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java @@ -2149,7 +2149,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac CompactionUtils.combineModsInInnerCompaction(sourceFiles, targetResources.get(0)); tsFileManager.replace(sourceFiles, Collections.emptyList(), targetResources, 0, true); CompactionUtils.deleteTsFilesInDisk(sourceFiles, COMPACTION_TEST_SG + "-" + "0"); - targetResources.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + targetResources.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); // start selecting files and then start a cross space compaction task ICrossSpaceSelector selector = @@ -2289,7 +2289,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac Assert.assertEquals(0, innerSelector.selectInnerSpaceTask(targetResources).size()); // first compaction task finishes successfully - targetResources.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + targetResources.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL)); // target file of first compaction task can be selected to participate in another cross // compaction task diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java index 27ecf3f6fc..6f4e440ab2 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java @@ -436,7 +436,7 @@ public class CrossSpaceCompactionWithReadPointPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); @@ -740,7 +740,7 @@ public class CrossSpaceCompactionWithReadPointPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); @@ -1043,7 +1043,7 @@ public class CrossSpaceCompactionWithReadPointPerformerTest { new TsFileResource( TsFileNameGenerator.increaseCrossCompactionCnt(seqResource).getTsFile()); targetResource.deserialize(); - targetResource.setStatus(TsFileResourceStatus.CLOSED); + targetResource.setStatus(TsFileResourceStatus.NORMAL); targetTsfileResourceList.add(targetResource); } CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsfileResourceList); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeTest.java index 9d2c4e4194..fb3b7d6549 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeTest.java @@ -104,7 +104,7 @@ abstract class MergeTest { File file = new File(TestConstant.getTestTsFilePath("root.sg1", 0, 0, i)); mkdirs(file); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i); tsFileResource.setMaxPlanIndex(i); tsFileResource.setVersion(i); @@ -115,7 +115,7 @@ abstract class MergeTest { File file = new File(TestConstant.getTestTsFilePath("root.sg1", 0, 0, i + seqFileNum)); mkdirs(file); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i + seqFileNum); tsFileResource.setMaxPlanIndex(i + seqFileNum); tsFileResource.setVersion(i + seqFileNum); @@ -127,7 +127,7 @@ abstract class MergeTest { new File(TestConstant.getTestTsFilePath("root.sg1", 0, 0, seqFileNum + unseqFileNum)); mkdirs(file); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setMaxPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setVersion(seqFileNum + unseqFileNum); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java index eb41d9a802..c216d52928 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java @@ -149,7 +149,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + ".tsfile")); TsFileResource largeUnseqTsFileResource = new TsFileResource(file); unseqResources.add(largeUnseqTsFileResource); - largeUnseqTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + largeUnseqTsFileResource.setStatus(TsFileResourceStatus.NORMAL); largeUnseqTsFileResource.setMinPlanIndex(10); largeUnseqTsFileResource.setMaxPlanIndex(10); largeUnseqTsFileResource.setVersion(10); @@ -200,7 +200,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + ".tsfile")); TsFileResource largeUnseqTsFileResource = new TsFileResource(file); unseqResources.add(largeUnseqTsFileResource); - largeUnseqTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + largeUnseqTsFileResource.setStatus(TsFileResourceStatus.NORMAL); largeUnseqTsFileResource.setMinPlanIndex(10); largeUnseqTsFileResource.setMaxPlanIndex(10); largeUnseqTsFileResource.setVersion(10); @@ -250,7 +250,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource largeUnseqTsFileResource = new TsFileResource(file); - largeUnseqTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + largeUnseqTsFileResource.setStatus(TsFileResourceStatus.NORMAL); largeUnseqTsFileResource.setMinPlanIndex(10); largeUnseqTsFileResource.setMaxPlanIndex(10); largeUnseqTsFileResource.setVersion(10); @@ -289,7 +289,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, 0); seqList.add(fileResource); } @@ -308,7 +308,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); unseqList.add(fileResource); } prepareFile(unseqList.get(0), 0, 1, 10); @@ -372,7 +372,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, i); seqList.add(fileResource); } @@ -391,7 +391,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, i); unseqList.add(fileResource); } @@ -445,7 +445,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, i); seqList.add(fileResource); } @@ -464,7 +464,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 10, i); unseqList.add(fileResource); } @@ -517,7 +517,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, i); seqList.add(fileResource); } @@ -536,7 +536,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); unseqList.add(fileResource); } prepareFile(unseqList.get(0), 7, 3, 7); @@ -591,7 +591,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); prepareFile(fileResource, i, 1, i); seqList.add(fileResource); } @@ -610,7 +610,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); unseqList.add(fileResource); } prepareFile(unseqList.get(0), 7, 3, 7); @@ -669,7 +669,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); if (i - 11 != 3) { - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); } prepareFile(fileResource, i, 1, i); seqList.add(fileResource); @@ -689,7 +689,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fileResource = new TsFileResource(file); - fileResource.setStatus(TsFileResourceStatus.CLOSED); + fileResource.setStatus(TsFileResourceStatus.NORMAL); unseqList.add(fileResource); } prepareFile(unseqList.get(0), 7, 3, 7); @@ -740,7 +740,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource firstTsFileResource = new TsFileResource(firstFile); - firstTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + firstTsFileResource.setStatus(TsFileResourceStatus.NORMAL); firstTsFileResource.setMinPlanIndex(1); firstTsFileResource.setMaxPlanIndex(1); firstTsFileResource.setVersion(1); @@ -793,7 +793,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource secondTsFileResource = new TsFileResource(secondFile); - secondTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + secondTsFileResource.setStatus(TsFileResourceStatus.NORMAL); secondTsFileResource.setMinPlanIndex(2); secondTsFileResource.setMaxPlanIndex(2); secondTsFileResource.setVersion(2); @@ -839,7 +839,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource thirdTsFileResource = new TsFileResource(thirdFile); - thirdTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + thirdTsFileResource.setStatus(TsFileResourceStatus.NORMAL); thirdTsFileResource.setMinPlanIndex(3); thirdTsFileResource.setMaxPlanIndex(3); thirdTsFileResource.setVersion(3); @@ -885,7 +885,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { + 0 + ".tsfile")); TsFileResource fourthTsFileResource = new TsFileResource(fourthFile); - fourthTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + fourthTsFileResource.setStatus(TsFileResourceStatus.NORMAL); fourthTsFileResource.setMinPlanIndex(4); fourthTsFileResource.setMaxPlanIndex(4); fourthTsFileResource.setVersion(4); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTest.java index 686858a35f..91d329476d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTest.java @@ -160,7 +160,7 @@ public abstract class AbstractInnerSpaceCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); @@ -179,7 +179,7 @@ public abstract class AbstractInnerSpaceCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(i + seqFileNum); unseqResources.add(tsFileResource); prepareFile(tsFileResource, i * 2 * ptNum, ptNum * (i + 1) / unseqFileNum, 10000); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionMoreDataTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionMoreDataTest.java index 2aafe3010f..0fc1823d36 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionMoreDataTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionMoreDataTest.java @@ -93,7 +93,7 @@ public class InnerCompactionMoreDataTest extends InnerCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionTest.java index 22a5825edf..b32bb02355 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionTest.java @@ -116,7 +116,7 @@ public abstract class InnerCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); @@ -135,7 +135,7 @@ public abstract class InnerCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(i + seqFileNum); unseqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTest.java index 8211c71ebb..8f5f993e9c 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTest.java @@ -114,7 +114,7 @@ public class SizeTieredCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); @@ -133,7 +133,7 @@ public class SizeTieredCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(i + seqFileNum); unseqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000); @@ -152,7 +152,7 @@ public class SizeTieredCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes(seqFileNum + unseqFileNum); unseqResources.add(tsFileResource); prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000); @@ -227,7 +227,7 @@ public class SizeTieredCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource1 = new TsFileResource(file1); - tsFileResource1.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource1.setStatus(TsFileResourceStatus.NORMAL); tsFileResource1.updatePlanIndexes((long) 0); TsFileWriter fileWriter1 = new TsFileWriter(tsFileResource1.getTsFile()); fileWriter1.registerTimeseries( @@ -255,7 +255,7 @@ public class SizeTieredCompactionTest { + 0 + ".tsfile")); TsFileResource tsFileResource2 = new TsFileResource(file2); - tsFileResource2.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource2.setStatus(TsFileResourceStatus.NORMAL); tsFileResource2.updatePlanIndexes((long) 1); TsFileWriter fileWriter2 = new TsFileWriter(tsFileResource2.getTsFile()); fileWriter2.registerTimeseries( diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java index 3160f03839..adeb64d99c 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java @@ -456,7 +456,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -601,7 +601,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -744,7 +744,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -895,7 +895,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -1040,7 +1040,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); @@ -1193,7 +1193,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest { CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFileManager.replace( tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true); - tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.CLOSED); + tsFileManager.getTsFileList(true).get(0).setStatus(TsFileResourceStatus.NORMAL); validateSeqFiles(true); validateTargetDatas(sourceData, Collections.emptyList()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java index 8c9f5f8ff0..3a8b0e87a1 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java @@ -85,7 +85,7 @@ public class IoTDBSnapshotTest { resource.updateEndTime(testSgName + PATH_SEPARATOR + "d" + i, (i + 1) * 100); } resource.updatePlanIndexes(i); - resource.setStatus(TsFileResourceStatus.CLOSED); + resource.setStatus(TsFileResourceStatus.NORMAL); resource.serialize(); } return resources; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java index 178c02ae61..085c127937 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java @@ -35,7 +35,7 @@ public class FakedTsFileResource extends TsFileResource { public FakedTsFileResource(long tsFileSize, String name) { this.timeIndex = new FileTimeIndex(); this.tsFileSize = tsFileSize; - super.status = TsFileResourceStatus.CLOSED; + super.status = TsFileResourceStatus.NORMAL; fakeTsfileName = name; } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceTest.java index 06f3af6fdd..76cb6cff9a 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceTest.java @@ -55,7 +55,7 @@ public class TsFileResourceTest { deviceTimeIndex.updateEndTime("root.sg.d" + i, i + 1); }); tsFileResource.setTimeIndex(deviceTimeIndex); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/AlignedSeriesTestUtil.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/AlignedSeriesTestUtil.java index c6be93db8c..a563f727c5 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/AlignedSeriesTestUtil.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/AlignedSeriesTestUtil.java @@ -87,7 +87,7 @@ public class AlignedSeriesTestUtil { for (int i = 0; i < seqFileNum; i++) { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, i)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i); tsFileResource.setMaxPlanIndex(i); tsFileResource.setVersion(i); @@ -98,7 +98,7 @@ public class AlignedSeriesTestUtil { for (int i = 0; i < unseqFileNum; i++) { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, i + seqFileNum)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i + seqFileNum); tsFileResource.setMaxPlanIndex(i + seqFileNum); tsFileResource.setVersion(i + seqFileNum); @@ -114,7 +114,7 @@ public class AlignedSeriesTestUtil { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, seqFileNum + unseqFileNum)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setMaxPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setVersion(seqFileNum + unseqFileNum); diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java index de86d59076..180962b0c5 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java @@ -137,7 +137,7 @@ public class AlignedSeriesScanLimitOffsetPushDownTest { seqFileResource1.updateEndTime(TEST_DEVICE, 9); tsFileIOWriter.endFile(); } - seqFileResource1.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource1.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource1); // prepare file 2 @@ -205,7 +205,7 @@ public class AlignedSeriesScanLimitOffsetPushDownTest { seqFileResource2.updateEndTime(TEST_DEVICE, 19); tsFileIOWriter.endFile(); } - seqFileResource2.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource2.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource2); // prepare file 3 @@ -285,7 +285,7 @@ public class AlignedSeriesScanLimitOffsetPushDownTest { seqFileResource3.updateEndTime(TEST_DEVICE, 39); tsFileIOWriter.endFile(); } - seqFileResource3.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource3.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource3); // prepare file 4 @@ -380,7 +380,7 @@ public class AlignedSeriesScanLimitOffsetPushDownTest { seqFileResource4.updateEndTime(TEST_DEVICE, 79); tsFileIOWriter.endFile(); } - seqFileResource4.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource4.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource4); // prepare file 5 @@ -404,7 +404,7 @@ public class AlignedSeriesScanLimitOffsetPushDownTest { unseqFileResource5.updateEndTime(TEST_DEVICE, 89); tsFileIOWriter.endFile(); } - unseqFileResource5.setStatus(TsFileResourceStatus.CLOSED); + unseqFileResource5.setStatus(TsFileResourceStatus.NORMAL); unSeqResources.add(unseqFileResource5); } diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java index b331966809..58f0afbfc4 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java @@ -113,7 +113,7 @@ public class SeriesReaderTestUtil { for (int i = 0; i < seqFileNum; i++) { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, i)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i); tsFileResource.setMaxPlanIndex(i); tsFileResource.setVersion(i); @@ -123,7 +123,7 @@ public class SeriesReaderTestUtil { for (int i = 0; i < unseqFileNum; i++) { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, i + seqFileNum)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(i + seqFileNum); tsFileResource.setMaxPlanIndex(i + seqFileNum); tsFileResource.setVersion(i + seqFileNum); @@ -139,7 +139,7 @@ public class SeriesReaderTestUtil { File file = new File(TestConstant.getTestTsFilePath(sgName, 0, 0, seqFileNum + unseqFileNum)); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.setMinPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setMaxPlanIndex(seqFileNum + unseqFileNum); tsFileResource.setVersion(seqFileNum + unseqFileNum); diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesScanLimitOffsetPushDownTest.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesScanLimitOffsetPushDownTest.java index dced673713..07d998aed2 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesScanLimitOffsetPushDownTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesScanLimitOffsetPushDownTest.java @@ -118,7 +118,7 @@ public class SeriesScanLimitOffsetPushDownTest { seqFileResource1.updateEndTime(TEST_DEVICE, 9); tsFileIOWriter.endFile(); } - seqFileResource1.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource1.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource1); // prepare file 2 @@ -151,7 +151,7 @@ public class SeriesScanLimitOffsetPushDownTest { seqFileResource2.updateEndTime(TEST_DEVICE, 29); tsFileIOWriter.endFile(); } - seqFileResource2.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource2.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource2); // prepare file 3 @@ -186,7 +186,7 @@ public class SeriesScanLimitOffsetPushDownTest { seqFileResource3.updateEndTime(TEST_DEVICE, 59); tsFileIOWriter.endFile(); } - seqFileResource3.setStatus(TsFileResourceStatus.CLOSED); + seqFileResource3.setStatus(TsFileResourceStatus.NORMAL); seqResources.add(seqFileResource3); // prepare file 4 @@ -210,7 +210,7 @@ public class SeriesScanLimitOffsetPushDownTest { unseqFileResource4.updateEndTime(TEST_DEVICE, 69); tsFileIOWriter.endFile(); } - unseqFileResource4.setStatus(TsFileResourceStatus.CLOSED); + unseqFileResource4.setStatus(TsFileResourceStatus.NORMAL); unSeqResources.add(unseqFileResource4); } diff --git a/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java b/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java index 2bc98aabb2..da5a493d56 100644 --- a/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java @@ -174,7 +174,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) 0); prepareFile(tsFileResource, 0, ptNum, 0); long previousRamSize = tsFileResource.calculateRamSize(); @@ -201,7 +201,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) 0); prepareFile(tsFileResource, 0, ptNum, 0); assertEquals( @@ -228,7 +228,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) 0); prepareFile(tsFileResource, 0, ptNum, 0); assertEquals( @@ -258,7 +258,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource1 = new TsFileResource(file1); - tsFileResource1.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource1.setStatus(TsFileResourceStatus.NORMAL); tsFileResource1.updatePlanIndexes((long) 0); prepareFile(tsFileResource1, 0, ptNum, 0); assertEquals( @@ -282,7 +282,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource2 = new TsFileResource(file2); - tsFileResource2.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource2.setStatus(TsFileResourceStatus.NORMAL); tsFileResource2.updatePlanIndexes((long) 1); prepareFile(tsFileResource2, ptNum, ptNum, 0); assertEquals( @@ -313,7 +313,7 @@ public class ResourceManagerTest { + 0 + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.setStatus(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); assertEquals( TimeIndexLevel.DEVICE_TIME_INDEX,
