This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new 2b268d46f30 [To rel/1.2][IOTDB-6016]Release file num cost after cross
compaction task (#10246)
2b268d46f30 is described below
commit 2b268d46f3081a7dcb4fc7e13fd69de0641e8ad3
Author: 周沛辰 <[email protected]>
AuthorDate: Tue Jun 20 21:28:44 2023 +0800
[To rel/1.2][IOTDB-6016]Release file num cost after cross compaction task
(#10246)
---
.../execute/task/CrossSpaceCompactionTask.java | 3 ++
.../FastCrossCompactionPerformerTest.java | 37 ++++++++++++++++++++++
.../db/engine/compaction/MemoryControlTest.java | 11 +++++++
.../cross/CrossSpaceCompactionSelectorTest.java | 2 ++
4 files changed, 53 insertions(+)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 7ce24e887cb..8d80ef45e25 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -274,6 +274,9 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
true);
} finally {
SystemInfo.getInstance().resetCompactionMemoryCost(memoryCost);
+ SystemInfo.getInstance()
+ .decreaseCompactionFileNumCost(
+ selectedSequenceFiles.size() + selectedUnsequenceFiles.size());
releaseAllLocksAndResetStatus();
return isSuccess;
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
index fa236ea2da4..831e194d423 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ICompactionPerformer;
import
org.apache.iotdb.db.engine.compaction.execute.performer.impl.FastCompactionPerformer;
+import
org.apache.iotdb.db.engine.compaction.execute.task.CrossSpaceCompactionTask;
import
org.apache.iotdb.db.engine.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import
org.apache.iotdb.db.engine.compaction.execute.utils.reader.IDataBlockReader;
@@ -34,6 +35,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -58,6 +60,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
import static org.apache.iotdb.db.utils.EnvironmentUtils.TEST_QUERY_JOB_ID;
@@ -4025,6 +4028,40 @@ public class FastCrossCompactionPerformerTest extends
AbstractCompactionTest {
}
}
+ @Test
+ public void testReleaseFileNumAndMemoryAfterCrossTask()
+ throws IOException, MetadataException, WriteProcessException {
+ int oldMaxCrossCompactionCandidateFileNum =
+ SystemInfo.getInstance().getTotalFileLimitForCrossTask();
+ SystemInfo.getInstance().setTotalFileLimitForCrossTask(15);
+ SystemInfo.getInstance().getCompactionFileNumCost().set(0);
+ SystemInfo.getInstance().getCompactionMemoryCost().set(0);
+ try {
+ createFiles(6, 2, 3, 300, 0, 0, 50, 50, false, true);
+ createFiles(6, 2, 3, 300, 0, 0, 50, 50, false, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+ CrossSpaceCompactionTask task =
+ new CrossSpaceCompactionTask(
+ 0L,
+ tsFileManager,
+ seqResources,
+ unseqResources,
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ 1000,
+ 0);
+ Assert.assertTrue(task.setSourceFilesToCompactionCandidate());
+ boolean success = task.checkValidAndSetMerging();
+ Assert.assertTrue(success);
+ Assert.assertTrue(task.start());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ } finally {
+
SystemInfo.getInstance().setTotalFileLimitForCrossTask(oldMaxCrossCompactionCandidateFileNum);
+ }
+ }
+
private void validateSeqFiles() {
TsFileValidationTool.clearMap(true);
List<File> files = new ArrayList<>();
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
index 04d8d87a671..24c18395363 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
@@ -19,23 +19,34 @@
package org.apache.iotdb.db.engine.compaction;
+import org.apache.iotdb.commons.exception.MetadataException;
import
org.apache.iotdb.db.engine.compaction.execute.task.CrossSpaceCompactionTask;
import
org.apache.iotdb.db.engine.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.rescon.SystemInfo;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
public class MemoryControlTest {
+ @Before
+ public void setUp()
+ throws IOException, WriteProcessException, MetadataException,
InterruptedException {
+ SystemInfo.getInstance().getCompactionFileNumCost().set(0);
+ SystemInfo.getInstance().getCompactionMemoryCost().set(0);
+ }
+
@Test
public void testFailedToAllocateMemoryInCrossTask() throws Exception {
List<TsFileResource> sequenceFiles = new ArrayList<>();
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
index 6407418ea08..d4d4c851231 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
@@ -180,6 +180,8 @@ public class CrossSpaceCompactionSelectorTest extends
AbstractCompactionTest {
throws IOException, MetadataException, WriteProcessException {
int oldMaxFileNumForCompaction =
SystemInfo.getInstance().getTotalFileLimitForCrossTask();
SystemInfo.getInstance().setTotalFileLimitForCrossTask(1);
+ SystemInfo.getInstance().getCompactionFileNumCost().set(0);
+ SystemInfo.getInstance().getCompactionMemoryCost().set(0);
try {
createFiles(19, 2, 3, 50, 0, 10000, 50, 50, false, true);
createFiles(1, 2, 3, 3000, 0, 10000, 50, 50, false, false);
