This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 8687800 add enable unseq compaction (#1972)
8687800 is described below
commit 868780038b371f9d879e417ccc364771f42440fc
Author: zhanglingzhe0820 <[email protected]>
AuthorDate: Fri Nov 6 23:16:54 2020 +0800
add enable unseq compaction (#1972)
(cherry picked from commit c0d731dbd5380295df5036504b7524a2ade50bbf)
---
server/src/assembly/resources/conf/iotdb-engine.properties | 4 ++++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 ++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 ++++++----
.../compaction/level/LevelCompactionTsFileManagement.java | 6 ++++--
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index 69efa26..b0a691b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -277,6 +277,10 @@ default_fill_interval=-1
compaction_strategy=LEVEL_COMPACTION
# Works when the compaction_strategy is LEVEL_COMPACTION.
+# Whether to merge unseq files into seq files or not.
+enable_unseq_compaction = true
+
+# Works when the compaction_strategy is LEVEL_COMPACTION.
# The max seq file num of each level.
# When the num of files in one level exceeds this,
# the files in this level will merge to one and put to upper level.
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 5b0acb82..72718e2 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -308,6 +308,12 @@ public class IoTDBConfig {
/**
* Works when the compaction_strategy is LEVEL_COMPACTION.
+ * Whether to merge unseq files into seq files or not.
+ */
+ private boolean enableUnseqCompaction = true;
+
+ /**
+ * Works when the compaction_strategy is LEVEL_COMPACTION.
* The max seq file num of each level.
* When the num of files in one level exceeds this,
* the files in this level will merge to one and put to upper level.
@@ -1442,6 +1448,14 @@ public class IoTDBConfig {
this.compactionStrategy = compactionStrategy;
}
+ public boolean isEnableUnseqCompaction() {
+ return enableUnseqCompaction;
+ }
+
+ public void setEnableUnseqCompaction(boolean enableUnseqCompaction) {
+ this.enableUnseqCompaction = enableUnseqCompaction;
+ }
+
public int getSeqFileNumInEachLevel() {
return seqFileNumInEachLevel;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index de14763..d7a7058 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -307,6 +307,10 @@ public class IoTDBDescriptor {
.getProperty("compaction_strategy",
conf.getCompactionStrategy().toString())));
+ conf.setEnableUnseqCompaction(Boolean.parseBoolean(
+ properties.getProperty("enable_unseq_compaction",
+ Boolean.toString(conf.isEnableUnseqCompaction()))));
+
conf.setSeqLevelNum(Integer.parseInt(properties
.getProperty("seq_level_num",
Integer.toString(conf.getSeqLevelNum()))));
@@ -770,7 +774,8 @@ public class IoTDBDescriptor {
}
private void initMemoryAllocate(Properties properties) {
- String memoryAllocateProportion =
properties.getProperty("write_read_schema_free_memory_proportion");
+ String memoryAllocateProportion = properties
+ .getProperty("write_read_schema_free_memory_proportion");
if (memoryAllocateProportion != null) {
String[] proportions = memoryAllocateProportion.split(":");
int proportionSum = 0;
@@ -831,9 +836,6 @@ public class IoTDBDescriptor {
/**
* Get default encode algorithm by data type
- *
- * @param dataType
- * @return
*/
public TSEncoding getDefualtEncodingByType(TSDataType dataType) {
switch (dataType) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index cba29a5..2c47e3a 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -68,6 +68,8 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
private final int unseqFileNumInEachLevel =
IoTDBDescriptor.getInstance().getConfig()
.getSeqFileNumInEachLevel();
+ private final boolean enableUnseqCompaction =
IoTDBDescriptor.getInstance().getConfig()
+ .isEnableUnseqCompaction();
private final boolean isForceFullMerge =
IoTDBDescriptor.getInstance().getConfig()
.isForceFullMerge();
// First map is partition list; Second list is level list; Third list is
file list in level;
@@ -424,7 +426,7 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
protected void merge(long timePartition) {
merge(forkedSequenceTsFileResources, true, timePartition, seqLevelNum,
seqFileNumInEachLevel);
- if (unseqLevelNum <= 1 && forkedUnSequenceTsFileResources.size() > 0) {
+ if (enableUnseqCompaction && unseqLevelNum <= 1 &&
forkedUnSequenceTsFileResources.size() > 0) {
merge(isForceFullMerge, getTsFileList(true),
forkedUnSequenceTsFileResources.get(0),
Long.MAX_VALUE);
} else {
@@ -444,7 +446,7 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
for (int i = 0; i < currMaxLevel - 1; i++) {
if (currMaxFileNumInEachLevel <= mergeResources.get(i).size()) {
//level is numbered from 0
- if (!sequence && i == currMaxLevel - 2) {
+ if (enableUnseqCompaction && !sequence && i == currMaxLevel - 2) {
// do not merge current unseq file level to upper level and just
merge all of them to seq file
merge(isForceFullMerge, getTsFileList(true),
mergeResources.get(i), Long.MAX_VALUE);
} else {
