This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7c89c5c2296 Fix storage engine memory config initialization (#10077)
7c89c5c2296 is described below
commit 7c89c5c2296e061021d494cf3acaf1306ffe07c7
Author: Haonan <[email protected]>
AuthorDate: Thu Jun 8 19:35:32 2023 +0800
Fix storage engine memory config initialization (#10077)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 83 ++++++++++++++--------
2 files changed, 55 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 816916ac7b1..2871122afe0 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -154,7 +154,7 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;
/** The proportion of write memory for memtable */
- private double writeProportionForMemtable = 0.8;
+ private double writeProportionForMemtable = 0.76;
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
@@ -554,7 +554,7 @@ public class IoTDBConfig {
private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001;
/** Memory allocated proportion for time partition info */
- private long allocateMemoryForTimePartitionInfo =
allocateMemoryForStorageEngine * 50 / 1001;
+ private long allocateMemoryForTimePartitionInfo =
allocateMemoryForStorageEngine * 8 / 10 / 20;
/** Memory allocated proportion for wal pipe cache */
private long allocateMemoryForWALPipeCache = allocateMemoryForConsensus / 10;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7d3093d1ba0..ddb685cf94a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1767,36 +1767,59 @@ public class IoTDBDescriptor {
}
private void initStorageEngineAllocate(Properties properties) {
- String allocationRatio =
properties.getProperty("storage_engine_memory_proportion", "8:2");
- String[] proportions = allocationRatio.split(":");
- int proportionForWrite = Integer.parseInt(proportions[0].trim());
- int proportionForCompaction = Integer.parseInt(proportions[1].trim());
-
- double writeProportion =
- ((double) (proportionForWrite) / (double) (proportionForCompaction +
proportionForWrite));
-
- String allocationRatioForWrite =
properties.getProperty("write_memory_proportion", "19:1");
- proportions = allocationRatioForWrite.split(":");
- int proportionForMemTable = Integer.parseInt(proportions[0].trim());
- int proportionForTimePartitionInfo =
Integer.parseInt(proportions[1].trim());
-
- double memtableProportionForWrite =
- ((double) (proportionForMemTable)
- / (double) (proportionForMemTable +
proportionForTimePartitionInfo));
-
- double timePartitionInfoForWrite =
- ((double) (proportionForTimePartitionInfo)
- / (double) (proportionForMemTable +
proportionForTimePartitionInfo));
- conf.setWriteProportionForMemtable(writeProportion *
memtableProportionForWrite);
-
- conf.setAllocateMemoryForTimePartitionInfo(
- (long)
- ((writeProportion * timePartitionInfoForWrite)
- * conf.getAllocateMemoryForStorageEngine()));
-
- conf.setCompactionProportion(
- ((double) (proportionForCompaction)
- / (double) (proportionForCompaction + proportionForWrite)));
+ long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine();
+
+ int proportionSum = 10;
+ int writeProportion = 8;
+ int compactionProportion = 2;
+ int writeProportionSum = 20;
+ int memTableProportion = 19;
+ int timePartitionInfo = 1;
+
+ String storageMemoryAllocatePortion =
+ properties.getProperty("storage_engine_memory_proportion");
+ if (storageMemoryAllocatePortion != null) {
+ String[] proportions = storageMemoryAllocatePortion.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ proportionSum = loadedProportionSum;
+ writeProportion = Integer.parseInt(proportions[0].trim());
+ compactionProportion = Integer.parseInt(proportions[1].trim());
+ }
+ conf.setCompactionProportion((double) compactionProportion / (double)
proportionSum);
+ }
+
+ String allocationRatioForWrite =
properties.getProperty("write_memory_proportion");
+ if (allocationRatioForWrite != null) {
+ String[] proportions = allocationRatioForWrite.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ writeProportionSum = loadedProportionSum;
+ memTableProportion = Integer.parseInt(proportions[0].trim());
+ timePartitionInfo = Integer.parseInt(proportions[1].trim());
+ }
+ // memtableProportionForWrite = 19/20 default
+ double memtableProportionForWrite =
+ ((double) memTableProportion / (double) writeProportionSum);
+
+ // timePartitionInfoForWrite = 1/20 default
+ double timePartitionInfoForWrite = ((double) timePartitionInfo /
(double) writeProportionSum);
+ // proportionForWrite = 8/10 default
+ double proportionForWrite = ((double) (writeProportion) / (double)
proportionSum);
+ // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
+ conf.setWriteProportionForMemtable(proportionForWrite *
memtableProportionForWrite);
+ // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20
default
+ conf.setAllocateMemoryForTimePartitionInfo(
+ (long) ((proportionForWrite * timePartitionInfoForWrite) *
storageMemoryTotal));
+ }
}
private void initSchemaMemoryAllocate(Properties properties) {
