This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 01e487e33f7 DataNodeDevicePathCache should use free memory of memtable
(#12034)
01e487e33f7 is described below
commit 01e487e33f7886afdb5c8d3b73af227800ce5eed
Author: Haonan <[email protected]>
AuthorDate: Wed Feb 7 13:58:54 2024 +0800
DataNodeDevicePathCache should use free memory of memtable (#12034)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 ++---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 ++++++++++++++--------
.../cache/schema/DataNodeDevicePathCache.java | 1 +
.../resources/conf/iotdb-common.properties | 9 ++++---
4 files changed, 30 insertions(+), 16 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f9e69193622..854401d7b5e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -166,13 +166,13 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;
/** The proportion of write memory for memtable */
- private double writeProportionForMemtable = 0.72;
+ private double writeProportionForMemtable = 0.76;
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
- /** The proportion of write memory for device path cache */
- private double devicePathCacheProportion = 0.04;
+ /** The proportion of memtable memory for device path cache */
+ private double devicePathCacheProportion = 0.05;
/**
* If memory cost of data region increased more than proportion of
{@linkplain
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c451841d928..e4c5c243598 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -284,11 +284,27 @@ public class IoTDBDescriptor {
.getProperty("flush_proportion",
Double.toString(conf.getFlushProportion()))
.trim()));
- conf.setRejectProportion(
+ double rejectProportion =
Double.parseDouble(
properties
.getProperty("reject_proportion",
Double.toString(conf.getRejectProportion()))
- .trim()));
+ .trim());
+
+ double devicePathCacheProportion =
+ Double.parseDouble(
+ properties
+ .getProperty(
+ "device_path_cache_proportion",
+ Double.toString(conf.getDevicePathCacheProportion()))
+ .trim());
+
+ if (rejectProportion + devicePathCacheProportion >= 1) {
+ LOGGER.warn(
+ "The sum of write_memory_proportion and device_path_cache_proportion
is too large, use default values 0.8 and 0.05.");
+ } else {
+ conf.setRejectProportion(rejectProportion);
+ conf.setDevicePathCacheProportion(devicePathCacheProportion);
+ }
conf.setWriteMemoryVariationReportProportion(
Double.parseDouble(
@@ -1772,7 +1788,7 @@ public class IoTDBDescriptor {
writeMemoryProportion += proportionValue;
if (proportionValue <= 0) {
LOGGER.warn(
- "The value of write_memory_proportion is illegal, use default
value 18:1:1 .");
+ "The value of write_memory_proportion is illegal, use default
value 19:1 .");
return;
}
}
@@ -1784,9 +1800,7 @@ public class IoTDBDescriptor {
(double) Integer.parseInt(writeProportionArray[0].trim()) /
writeMemoryProportion;
double timePartitionInfoProportion =
(double) Integer.parseInt(writeProportionArray[1].trim()) /
writeMemoryProportion;
- double devicePathCacheProportion =
- (double) Integer.parseInt(writeProportionArray[2].trim()) /
writeMemoryProportion;
- // writeProportionForMemtable = 8/10 * 18/20 = 0.72 default
+ // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
conf.setWriteProportionForMemtable(
writeAllProportionOfStorageEngineMemory * memTableProportion);
@@ -1795,10 +1809,6 @@ public class IoTDBDescriptor {
(long)
((writeAllProportionOfStorageEngineMemory *
timePartitionInfoProportion)
* storageMemoryTotal));
-
- // device path cache default memory is value is 8/10 * 1/20 = 0.04 for
StorageEngine
- conf.setDevicePathCacheProportion(
- writeAllProportionOfStorageEngineMemory *
devicePathCacheProportion);
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
index 0a189043e11..f444f9bb9e5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
@@ -41,6 +41,7 @@ public class DataNodeDevicePathCache {
.maximumWeight(
(long)
(config.getAllocateMemoryForStorageEngine()
+ * config.getWriteProportionForMemtable()
* config.getDevicePathCacheProportion()))
.weigher(
(Weigher<String, PartialPath>) (key, val) ->
(PartialPath.estimateSize(val) + 32))
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 5330bdaa44d..88d1d0f47cf 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -181,11 +181,10 @@ data_replication_factor=1
# The parameter form is a:b:c:d, where a, b, c and d are integers. for
example: 8:2 , 7:3
# storage_engine_memory_proportion=8:2
-# Memory allocation ratio in writing: Memtable, TimePartitionInfo,
DevicePathCache
+# Memory allocation ratio in writing: Memtable, TimePartitionInfo
# Memtable is the total memory size of all memtables
# TimePartitionInfo is the total memory size of last flush time of all data
regions
-# DevicePathCache is the deviceId cache, keep only one copy of the same
deviceId in memory
-# write_memory_proportion=18:1:1
+# write_memory_proportion=19:1
# primitive array size (length of each array) in array pool
# Datatype: int
@@ -210,6 +209,10 @@ data_replication_factor=1
# Datatype: double
# reject_proportion=0.8
+# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId
cache, keep only one copy of the same deviceId in memory
+# Datatype: double
+# device_path_cache_proportion=0.05
+
# If memory cost of data region increased more than proportion of allocated
memory for write, report to system. The default value is 0.001
# Datatype: double
# write_memory_variation_report_proportion=0.001
