This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch DataNodeDevicePathCache_use_free_memory in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0f98b9349645bd688cabd1931058f8d7c30d4f84 Author: HTHou <[email protected]> AuthorDate: Wed Feb 7 11:37:00 2024 +0800 DataNodeDevicePathCache should use free memory of memtable --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 ++-- .../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 ++++++++++-------- .../analyze/cache/schema/DataNodeDevicePathCache.java | 1 + .../assembly/resources/conf/iotdb-common.properties | 9 ++++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f9e69193622..f028ca4cb12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -166,13 +166,13 @@ public class IoTDBConfig { private double rejectProportion = 0.8; /** The proportion of write memory for memtable */ - private double writeProportionForMemtable = 0.72; + private double writeProportionForMemtable = 0.76; /** The proportion of write memory for compaction */ private double compactionProportion = 0.2; /** The proportion of write memory for device path cache */ - private double devicePathCacheProportion = 0.04; + private double devicePathCacheProportion = 0.05; /** * If memory cost of data region increased more than proportion of {@linkplain diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c451841d928..146995a8846 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -290,6 +290,14 @@ public class IoTDBDescriptor { .getProperty("reject_proportion", Double.toString(conf.getRejectProportion())) .trim())); + conf.setDevicePathCacheProportion( + Double.parseDouble( + properties + .getProperty( + "device_path_cache_proportion", + Double.toString(conf.getDevicePathCacheProportion())) + .trim())); + conf.setWriteMemoryVariationReportProportion( Double.parseDouble( properties @@ -1772,7 +1780,7 @@ public class IoTDBDescriptor { writeMemoryProportion += proportionValue; if (proportionValue <= 0) { LOGGER.warn( - "The value of write_memory_proportion is illegal, use default value 18:1:1 ."); + "The value of write_memory_proportion is illegal, use default value 19:1 ."); return; } } @@ -1784,9 +1792,7 @@ public class IoTDBDescriptor { (double) Integer.parseInt(writeProportionArray[0].trim()) / writeMemoryProportion; double timePartitionInfoProportion = (double) Integer.parseInt(writeProportionArray[1].trim()) / writeMemoryProportion; - double devicePathCacheProportion = - (double) Integer.parseInt(writeProportionArray[2].trim()) / writeMemoryProportion; - // writeProportionForMemtable = 8/10 * 18/20 = 0.72 default + // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default conf.setWriteProportionForMemtable( writeAllProportionOfStorageEngineMemory * memTableProportion); @@ -1795,10 +1801,6 @@ public class IoTDBDescriptor { (long) ((writeAllProportionOfStorageEngineMemory * timePartitionInfoProportion) * storageMemoryTotal)); - - // device path cache default memory is value is 8/10 * 1/20 = 0.04 for StorageEngine - conf.setDevicePathCacheProportion( - writeAllProportionOfStorageEngineMemory * devicePathCacheProportion); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java index 0a189043e11..f444f9bb9e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java @@ -41,6 +41,7 @@ public class DataNodeDevicePathCache { .maximumWeight( (long) (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() * config.getDevicePathCacheProportion())) .weigher( (Weigher<String, PartialPath>) (key, val) -> (PartialPath.estimateSize(val) + 32)) diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index 5330bdaa44d..88d1d0f47cf 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -181,11 +181,10 @@ data_replication_factor=1 # The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 # storage_engine_memory_proportion=8:2 -# Memory allocation ratio in writing: Memtable, TimePartitionInfo, DevicePathCache +# Memory allocation ratio in writing: Memtable, TimePartitionInfo # Memtable is the total memory size of all memtables # TimePartitionInfo is the total memory size of last flush time of all data regions -# DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory -# write_memory_proportion=18:1:1 +# write_memory_proportion=19:1 # primitive array size (length of each array) in array pool # Datatype: int @@ -210,6 +209,10 @@ data_replication_factor=1 # Datatype: double # reject_proportion=0.8 +# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory +# Datatype: double +# device_path_cache_proportion=0.05 + # If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001 # Datatype: double # write_memory_variation_report_proportion=0.001
