This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4f80becac866c5d01c1e6f4c21b663aa373764a0 Author: Haonan <[email protected]> AuthorDate: Wed Feb 7 13:58:54 2024 +0800 DataNodeDevicePathCache should use free memory of memtable (#12034) --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 ++--- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 ++++++++++++++-------- .../cache/schema/DataNodeDevicePathCache.java | 1 + .../resources/conf/iotdb-common.properties | 9 ++++--- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 431cfe3fdca..e997b042547 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -166,13 +166,13 @@ public class IoTDBConfig { private double rejectProportion = 0.8; /** The proportion of write memory for memtable */ - private double writeProportionForMemtable = 0.72; + private double writeProportionForMemtable = 0.76; /** The proportion of write memory for compaction */ private double compactionProportion = 0.2; - /** The proportion of write memory for device path cache */ - private double devicePathCacheProportion = 0.04; + /** The proportion of memtable memory for device path cache */ + private double devicePathCacheProportion = 0.05; /** * If memory cost of data region increased more than proportion of {@linkplain diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index fc9bb6c2364..edde740a339 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -285,11 +285,27 @@ public class IoTDBDescriptor { .getProperty("flush_proportion", Double.toString(conf.getFlushProportion())) .trim())); - conf.setRejectProportion( + double rejectProportion = Double.parseDouble( properties .getProperty("reject_proportion", Double.toString(conf.getRejectProportion())) - .trim())); + .trim()); + + double devicePathCacheProportion = + Double.parseDouble( + properties + .getProperty( + "device_path_cache_proportion", + Double.toString(conf.getDevicePathCacheProportion())) + .trim()); + + if (rejectProportion + devicePathCacheProportion >= 1) { + LOGGER.warn( + "The sum of write_memory_proportion and device_path_cache_proportion is too large, use default values 0.8 and 0.05."); + } else { + conf.setRejectProportion(rejectProportion); + conf.setDevicePathCacheProportion(devicePathCacheProportion); + } conf.setWriteMemoryVariationReportProportion( Double.parseDouble( @@ -1830,7 +1846,7 @@ public class IoTDBDescriptor { writeMemoryProportion += proportionValue; if (proportionValue <= 0) { LOGGER.warn( - "The value of write_memory_proportion is illegal, use default value 18:1:1 ."); + "The value of write_memory_proportion is illegal, use default value 19:1 ."); return; } } @@ -1842,9 +1858,7 @@ public class IoTDBDescriptor { (double) Integer.parseInt(writeProportionArray[0].trim()) / writeMemoryProportion; double timePartitionInfoProportion = (double) Integer.parseInt(writeProportionArray[1].trim()) / writeMemoryProportion; - double devicePathCacheProportion = - (double) Integer.parseInt(writeProportionArray[2].trim()) / writeMemoryProportion; - // writeProportionForMemtable = 8/10 * 18/20 = 0.72 default + // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default conf.setWriteProportionForMemtable( writeAllProportionOfStorageEngineMemory * memTableProportion); @@ -1853,10 +1867,6 @@ public class IoTDBDescriptor { (long) ((writeAllProportionOfStorageEngineMemory * timePartitionInfoProportion) * storageMemoryTotal)); - - // device path cache default memory is value is 8/10 * 1/20 = 0.04 for StorageEngine - conf.setDevicePathCacheProportion( - writeAllProportionOfStorageEngineMemory * devicePathCacheProportion); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java index 0a189043e11..f444f9bb9e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java @@ -41,6 +41,7 @@ public class DataNodeDevicePathCache { .maximumWeight( (long) (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() * config.getDevicePathCacheProportion())) .weigher( (Weigher<String, PartialPath>) (key, val) -> (PartialPath.estimateSize(val) + 32)) diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index 07e8848eea8..81e0daa33bd 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -185,11 +185,10 @@ data_replication_factor=1 # The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 # storage_engine_memory_proportion=8:2 -# Memory allocation ratio in writing: Memtable, TimePartitionInfo, DevicePathCache +# Memory allocation ratio in writing: Memtable, TimePartitionInfo # Memtable is the total memory size of all memtables # TimePartitionInfo is the total memory size of last flush time of all data regions -# DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory -# write_memory_proportion=18:1:1 +# write_memory_proportion=19:1 # Max number of concurrent writing time partitions in one database # This parameter is used to control total memTable number when memory control is disabled @@ -220,6 +219,10 @@ data_replication_factor=1 # Datatype: double # reject_proportion=0.8 +# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory +# Datatype: double +# device_path_cache_proportion=0.05 + # If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001 # Datatype: double # write_memory_variation_report_proportion=0.001
