This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f34c20cb05 [IOTDB-4166] Fix select no files when time partition
interval is too small (#7036)
f34c20cb05 is described below
commit f34c20cb0518fbbdc8300396ebc33967207bd499
Author: Liu Xuxin <[email protected]>
AuthorDate: Thu Aug 18 07:45:14 2022 +0800
[IOTDB-4166] Fix select no files when time partition interval is too small
(#7036)
* Fix select no files when time partition interval is too small
* rename virable name
---
.../db/engine/compaction/CompactionScheduler.java | 6 +-
.../constant/InnerSequenceCompactionSelector.java | 8 ++-
.../InnerUnsequenceCompactionSelector.java | 8 ++-
.../sizetiered/SizeTieredCompactionSelector.java | 22 +++++++-
.../db/engine/storagegroup/TsFileManager.java | 14 ++++-
.../SizeTieredCompactionSelectorTest.java | 66 ++++++++++++++++++++++
.../engine/storagegroup/FakedTsFileResource.java | 9 ++-
7 files changed, 120 insertions(+), 13 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
index cc4ad3d66e..fd4bcf1c04 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
@@ -80,7 +80,7 @@ public class CompactionScheduler {
}
public static void tryToSubmitInnerSpaceCompactionTask(
- String logicalStorageGroupName,
+ String storageGroupName,
String dataRegionId,
long timePartition,
TsFileManager tsFileManager,
@@ -96,12 +96,12 @@ public class CompactionScheduler {
innerSpaceCompactionSelector =
config
.getInnerSequenceCompactionSelector()
- .createInstance(logicalStorageGroupName, dataRegionId,
timePartition);
+ .createInstance(storageGroupName, dataRegionId, timePartition,
tsFileManager);
} else {
innerSpaceCompactionSelector =
config
.getInnerUnsequenceCompactionSelector()
- .createInstance(logicalStorageGroupName, dataRegionId,
timePartition);
+ .createInstance(storageGroupName, dataRegionId, timePartition,
tsFileManager);
}
List<List<TsFileResource>> taskList =
innerSpaceCompactionSelector.selectInnerSpaceTask(
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java
index a40fefa08d..05949829a0 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction.constant;
import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector;
import
org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
public enum InnerSequenceCompactionSelector {
SIZE_TIERED;
@@ -33,12 +34,15 @@ public enum InnerSequenceCompactionSelector {
}
public IInnerSeqSpaceSelector createInstance(
- String logicalStorageGroupName, String virtualStorageGroupName, long
timePartition) {
+ String storageGroupName,
+ String dataRegionId,
+ long timePartition,
+ TsFileManager tsFileManager) {
switch (this) {
case SIZE_TIERED:
default:
return new SizeTieredCompactionSelector(
- logicalStorageGroupName, virtualStorageGroupName, timePartition,
true);
+ storageGroupName, dataRegionId, timePartition, true,
tsFileManager);
}
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java
index 5c5698f51d..983e89d64e 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.compaction.constant;
import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector;
import
org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
public enum InnerUnsequenceCompactionSelector {
SIZE_TIERED;
@@ -33,12 +34,15 @@ public enum InnerUnsequenceCompactionSelector {
}
public IInnerUnseqSpaceSelector createInstance(
- String logicalStorageGroupName, String virtualStorageGroupName, long
timePartition) {
+ String storageGroupName,
+ String dataRegionId,
+ long timePartition,
+ TsFileManager tsFileManager) {
switch (this) {
case SIZE_TIERED:
default:
return new SizeTieredCompactionSelector(
- logicalStorageGroupName, virtualStorageGroupName, timePartition,
false);
+ storageGroupName, dataRegionId, timePartition, false,
tsFileManager);
}
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
index bb16f60c5a..6261787398 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionTaskManager;
import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector;
import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
@@ -55,18 +56,26 @@ public class SizeTieredCompactionSelector
private static final Logger LOGGER =
LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
- protected String logicalStorageGroupName;
+ protected String storageGroupName;
protected String dataRegionId;
protected long timePartition;
protected List<TsFileResource> tsFileResources;
protected boolean sequence;
+ protected TsFileManager tsFileManager;
+ protected boolean hasNextTimePartition;
public SizeTieredCompactionSelector(
- String logicalStorageGroupName, String dataRegionId, long timePartition,
boolean sequence) {
- this.logicalStorageGroupName = logicalStorageGroupName;
+ String storageGroupName,
+ String dataRegionId,
+ long timePartition,
+ boolean sequence,
+ TsFileManager tsFileManager) {
+ this.storageGroupName = storageGroupName;
this.dataRegionId = dataRegionId;
this.timePartition = timePartition;
this.sequence = sequence;
+ this.tsFileManager = tsFileManager;
+ hasNextTimePartition = tsFileManager.hasNextTimePartition(timePartition,
sequence);
}
/**
@@ -152,6 +161,13 @@ public class SizeTieredCompactionSelector
shouldContinueToSearch = false;
}
}
+
+ // if next time partition exists
+ // submit a merge task even it does not meet the requirement for file num
or file size
+ if (hasNextTimePartition && selectedFileList.size() > 1) {
+ taskPriorityQueue.add(new Pair<>(new ArrayList<>(selectedFileList),
selectedFileSize));
+ shouldContinueToSearch = false;
+ }
return shouldContinueToSearch;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
index 11df2f8b4b..f09347c8d1 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
@@ -54,8 +54,8 @@ public class TsFileManager {
private String writeLockHolder;
// time partition -> double linked list of tsfiles
- private Map<Long, TsFileResourceList> sequenceFiles = new TreeMap<>();
- private Map<Long, TsFileResourceList> unsequenceFiles = new TreeMap<>();
+ private TreeMap<Long, TsFileResourceList> sequenceFiles = new TreeMap<>();
+ private TreeMap<Long, TsFileResourceList> unsequenceFiles = new TreeMap<>();
private List<TsFileResource> sequenceRecoverTsFileResources = new
ArrayList<>();
private List<TsFileResource> unsequenceRecoverTsFileResources = new
ArrayList<>();
@@ -431,4 +431,14 @@ public class TsFileManager {
public long getNextCompactionTaskId() {
return currentCompactionTaskSerialId.getAndIncrement();
}
+
+ public boolean hasNextTimePartition(long timePartition, boolean sequence) {
+ try {
+ return sequence
+ ? sequenceFiles.higherKey(timePartition) != null
+ : unsequenceFiles.higherKey(timePartition) != null;
+ } catch (NullPointerException e) {
+ return false;
+ }
+ }
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
new file mode 100644
index 0000000000..0946d0c7eb
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.compaction.inner.sizetiered;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.storagegroup.FakedTsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SizeTieredCompactionSelectorTest {
+ @Test
+ public void testSubmitWhenNextTimePartitionExists() {
+ long originPartitionInterval =
IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
+ IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(1000);
+ List<TsFileResource> resources = new ArrayList<>();
+
+ for (int i = 0; i < 100; ++i) {
+ FakedTsFileResource resource =
+ new FakedTsFileResource(1024, String.format("%d-%d-0-0.tsfile", i +
1, i + 1));
+ resource.timeIndex.updateStartTime("root.test.d", i * 100);
+ resource.timeIndex.updateEndTime("root.test.d", (i + 1) * 100);
+ resource.timePartition = i / 10;
+ resources.add(resource);
+ }
+
+ TsFileManager manager = new TsFileManager("root.test", "0", "");
+ manager.addAll(resources, true);
+
+ for (long i = 0; i < 9; ++i) {
+ Assert.assertEquals(
+ 1,
+ new SizeTieredCompactionSelector("root.test", "0", i, true, manager)
+ .selectInnerSpaceTask(manager.getSequenceListByTimePartition(i))
+ .size());
+ }
+
+ Assert.assertEquals(
+ 0,
+ new SizeTieredCompactionSelector("root.test", "0", 9, true, manager)
+ .selectInnerSpaceTask(manager.getSequenceListByTimePartition(9))
+ .size());
+ }
+}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java
index 5ff351c817..178c02ae61 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FakedTsFileResource.java
@@ -25,7 +25,9 @@ import java.io.File;
public class FakedTsFileResource extends TsFileResource {
/** time index */
- protected ITimeIndex timeIndex;
+ public ITimeIndex timeIndex;
+
+ public long timePartition;
private long tsFileSize;
private String fakeTsfileName;
@@ -69,4 +71,9 @@ public class FakedTsFileResource extends TsFileResource {
return false;
}
+
+ @Override
+ public long getTimePartition() {
+ return this.timePartition;
+ }
}
