This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1cd0caf3de [IOTDB-5871]Prevent large seq files from participating
cross space compaction (#9842)
1cd0caf3de is described below
commit 1cd0caf3dea806ed86ee0c028fc2a6810dbf2c9f
Author: 周沛辰 <[email protected]>
AuthorDate: Thu May 18 10:46:30 2023 +0800
[IOTDB-5871]Prevent large seq files from participating cross space
compaction (#9842)
---
docs/UserGuide/Reference/Common-Config-Manual.md | 12 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 8 +-
.../resources/conf/iotdb-common.properties | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 26 +++-
.../utils/CrossSpaceCompactionCandidate.java | 2 +-
.../db/tools/validate/TsFileValidationTool.java | 16 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 171 +++++++++++++++++++++
.../cross/RewriteCompactionFileSelectorTest.java | 8 +-
9 files changed, 221 insertions(+), 29 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md
b/docs/UserGuide/Reference/Common-Config-Manual.md
index 539cc5f1c5..1f098ba49a 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -911,12 +911,12 @@ Different configuration parameters take effect in the
following three ways:
* target\_compaction\_file\_size
-| Name | target\_compaction\_file\_size |
-| :---------: | :------------------------------------------- |
-| Description | The target file is in inner space compaction |
-| Type | Int64 |
-| Default | 1073741824 |
-| Effective | After restart system |
+| Name | target\_compaction\_file\_size |
+| :---------: |:-----------------------------------------------|
+| Description | The target file size in compaction |
+| Type | Int64 |
+| Default | 2147483648 |
+| Effective | After restart system |
* target\_chunk\_size
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 423266344a..0db310901e 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -960,11 +960,11 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
* target\_compaction\_file\_size
| 名字 | target\_compaction\_file\_size |
-| :----------: | :----------------------------- |
-| 描述 | 空间内合并的目标文件大小 |
+| :----------: |:-------------------------------|
+| 描述 | 合并后的目标文件大小 |
| 类型 | Int64 |
-| 默认值 | 1073741824 |
-| 改后生效方式 | 重启服务生效 |
+| 默认值 | 2147483648 |
+| 改后生效方式 | 重启服务生效 |
* target\_chunk\_size
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 0ecccf5cec..f492bf2e99 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -609,8 +609,9 @@ cluster_name=defaultCluster
# candidate_compaction_task_queue_size = 50
# The target tsfile size in compaction
+# default is 2GB
# Datatype: long, Unit: byte
-# target_compaction_file_size=1073741824
+# target_compaction_file_size=2147483648
# The target chunk size in compaction and when memtable reaches this
threshold, flush the memtable to disk.
# default is 1MB
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 76fce18d05..7b3ec1b493 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -464,8 +464,8 @@ public class IoTDBConfig {
private double chunkMetadataSizeProportion = 0.1;
- /** The target tsfile size in compaction, 1 GB by default */
- private long targetCompactionFileSize = 1073741824L;
+ /** The target tsfile size in compaction, 2 GB by default */
+ private long targetCompactionFileSize = 2147483648L;
/** The target chunk size in compaction. */
private long targetChunkSize = 1048576L;
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index 029b56d3e5..f0d3021307 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -157,12 +157,14 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
if (!split.hasOverlap) {
LOGGER.info("Unseq file {} does not overlap with any seq files.",
unseqFile);
- TsFileResource latestSealedSeqFile =
+ TsFileResourceCandidate latestSealedSeqFile =
getLatestSealedSeqFile(candidate.getSeqFileCandidates());
if (latestSealedSeqFile == null) {
break;
}
- targetSeqFiles.add(latestSealedSeqFile);
+ if (!latestSealedSeqFile.selected) {
+ targetSeqFiles.add(latestSealedSeqFile.resource);
+ }
}
long memoryCost =
@@ -182,7 +184,7 @@ public class RewriteCrossSpaceCompactionSelector implements
ICrossSpaceSelector
return taskResource;
}
- private TsFileResource getLatestSealedSeqFile(
+ private TsFileResourceCandidate getLatestSealedSeqFile(
List<TsFileResourceCandidate> seqResourceCandidateList) {
for (int i = seqResourceCandidateList.size() - 1; i >= 0; i--) {
TsFileResourceCandidate seqResourceCandidate =
seqResourceCandidateList.get(i);
@@ -193,7 +195,7 @@ public class RewriteCrossSpaceCompactionSelector implements
ICrossSpaceSelector
LOGGER.info(
"Select one valid seq file {} for unseq file to compact with.",
seqResourceCandidate.resource);
- return seqResourceCandidate.resource;
+ return seqResourceCandidate;
}
break;
}
@@ -220,16 +222,24 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
&& unseqFileName.getInnerCompactionCnt() <
config.getMinCrossCompactionUnseqFileLevel()) {
return false;
}
+
+ long totalFileSize = unseqFile.getTsFileSize();
+ for (TsFileResource f : seqFiles) {
+ if (f.getTsFileSize() >= config.getTargetCompactionFileSize()) {
+ // to avoid serious write amplification caused by cross space
compaction, we restrict that
+ // seq files are no longer be compacted when the size reaches the
threshold.
+ return false;
+ }
+ totalFileSize += f.getTsFileSize();
+ }
+
// currently, we must allow at least one unseqFile be selected to handle
the situation that
// an unseqFile has huge time range but few data points.
// IMPORTANT: this logic is opposite to previous level control
if (taskResource.getUnseqFiles().isEmpty()) {
return true;
}
- long totalFileSize = unseqFile.getTsFileSize();
- for (TsFileResource f : seqFiles) {
- totalFileSize += f.getTsFileSize();
- }
+
if (taskResource.getTotalFileNums() + 1 + seqFiles.size() <=
maxCrossCompactionFileNum
&& taskResource.getTotalFileSize() + totalFileSize <=
maxCrossCompactionFileSize
&& taskResource.getTotalMemoryCost() + memoryCost < memoryBudget) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
index bedf2e2fbc..1761b9a3b7 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
@@ -188,7 +188,7 @@ public class CrossSpaceCompactionCandidate {
public static class TsFileResourceCandidate {
public TsFileResource resource;
- protected boolean selected;
+ public boolean selected;
public boolean isValidCandidate;
private Map<String, DeviceInfo> deviceInfoMap;
diff --git
a/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
b/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
index 117d63d79d..abe6775d1b 100644
---
a/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
+++
b/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
@@ -158,10 +158,18 @@ public class TsFileValidationTool {
file -> file.getName().endsWith(TSFILE_SUFFIX))));
// sort the seq files with timestamp
tsFiles.sort(
- (f1, f2) ->
- Long.compareUnsigned(
- Long.parseLong(f1.getName().split("-")[0]),
- Long.parseLong(f2.getName().split("-")[0])));
+ (f1, f2) -> {
+ int timeDiff =
+ Long.compareUnsigned(
+ Long.parseLong(f1.getName().split("-")[0]),
+ Long.parseLong(f2.getName().split("-")[0]));
+ return timeDiff == 0
+ ? Long.compareUnsigned(
+ Long.parseLong(f1.getName().split("-")[1]),
+ Long.parseLong(f2.getName().split("-")[1]))
+ : timeDiff;
+ });
+
findUncorrectFiles(tsFiles);
}
// clear map
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
new file mode 100644
index 0000000000..7276670030
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.compaction.cross;
+
+import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
+import
org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector;
+import
org.apache.iotdb.db.engine.compaction.selector.utils.CrossCompactionTaskResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CrossSpaceCompactionSelectorTest extends AbstractCompactionTest {
+
+ @Before
+ public void setUp()
+ throws IOException, WriteProcessException, MetadataException,
InterruptedException {
+ super.setUp();
+
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
+ }
+
+ @After
+ public void tearDown() throws IOException, StorageEngineException {
+ super.tearDown();
+ for (TsFileResource tsFileResource : seqResources) {
+
FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFilePath());
+ }
+ for (TsFileResource tsFileResource : unseqResources) {
+
FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFilePath());
+ }
+ }
+
+ @Test
+ public void testSelectWithEmptySeqFileList()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(5, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(0, selected.size());
+ }
+
+ @Test
+ public void testSelectWithOneUnclosedSeqFile()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(5, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ seqResources.get(0).setStatus(TsFileResourceStatus.UNCLOSED);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(0, selected.size());
+ }
+
+ @Test
+ public void testSelectWithClosedSeqFileAndUnOverlapUnseqFile()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(5, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(1, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(5, selected.get(0).getUnseqFiles().size());
+
+ createFiles(1, 2, 3, 50, 100, 10000, 50, 50, false, true);
+ selected = selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(2, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(5, selected.get(0).getUnseqFiles().size());
+ }
+
+ @Test
+ public void testSelectWithClosedSeqFileAndUncloseSeqFile()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(2, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(5, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ seqResources.get(1).setStatus(TsFileResourceStatus.UNCLOSED);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(1, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(1, selected.get(0).getUnseqFiles().size());
+
+ createFiles(1, 2, 3, 200, 200, 10000, 50, 50, false, true);
+ seqResources.get(1).setStatus(TsFileResourceStatus.CLOSED);
+ seqResources.get(2).setStatus(TsFileResourceStatus.UNCLOSED);
+ selected = selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(2, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(2, selected.get(0).getUnseqFiles().size());
+
+ createFiles(1, 2, 3, 200, 1000, 10000, 50, 50, false, true);
+ createFiles(1, 2, 3, 200, 2000, 10000, 50, 50, false, true);
+ seqResources.get(2).setStatus(TsFileResourceStatus.CLOSED);
+ seqResources.get(4).setStatus(TsFileResourceStatus.UNCLOSED);
+ selected = selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(4, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(5, selected.get(0).getUnseqFiles().size());
+ }
+
+ @Test
+ public void testSelectWithMultiUnseqFilesOverlapWithOneSeqFile()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(3, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ createFiles(1, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(1, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(5, selected.get(0).getUnseqFiles().size());
+ }
+
+ @Test
+ public void testSelectWithTooLargeSeqFile()
+ throws IOException, MetadataException, WriteProcessException {
+ createFiles(2, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(5, 2, 3, 50, 0, 10000, 50, 50, false, false);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(2, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(5, selected.get(0).getUnseqFiles().size());
+
+ IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1L);
+ selected = selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(0, selected.size());
+ }
+}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
index eb41d9a802..9f65b157e3 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
@@ -1028,13 +1028,15 @@ public class RewriteCompactionFileSelectorTest extends
MergeTest {
}
@Test
- public void testFirstUnseqFileIsLarge() {
+ public void testFirstZeroLevelUnseqFileIsLarge() {
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(1);
-
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1024);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setTargetCompactionFileSize(unseqResources.get(5).getTsFileSize());
RewriteCrossSpaceCompactionSelector selector =
new RewriteCrossSpaceCompactionSelector("", "", 0, null);
List<CrossCompactionTaskResource> selected =
- selector.selectCrossSpaceTask(seqResources, unseqResources);
+ selector.selectCrossSpaceTask(seqResources, unseqResources.subList(5,
6));
Assert.assertEquals(1, selected.size());
}
}
