This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch feature/memory_transfer in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 182f7ead1a62450505dd1bc1f7dedca9d26f004e Author: spricoder <[email protected]> AuthorDate: Sat Feb 22 15:58:35 2025 +0800 Update Runtime.maxMemory() to Runtime().totalMemory() --- .../iotdb/consensus/config/IoTConsensusConfig.java | 2 +- .../logdispatcher/IoTConsensusMemoryManager.java | 2 +- .../src/assembly/resources/conf/datanode-env.sh | 7 ++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 66 +++++++++++----------- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java index 0621aee23ef..c01553c1391 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java @@ -359,7 +359,7 @@ public class IoTConsensusConfig { private long checkpointGap = 500; private IMemoryBlock consensusMemoryBlock = new AtomicLongMemoryBlock( - "Consensus-Default", null, Runtime.getRuntime().maxMemory() / 10); + "Consensus-Default", null, Runtime.getRuntime().totalMemory() / 10); private double maxMemoryRatioForQueue = 0.6; private long regionMigrationSpeedLimitBytesPerSecond = 32 * 1024 * 1024L; diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java index 7bee00588cd..4cb141f192f 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java @@ -33,7 +33,7 @@ public class IoTConsensusMemoryManager { private final AtomicLong queueMemorySizeInByte = new AtomicLong(0); private final AtomicLong syncMemorySizeInByte = new AtomicLong(0); private IMemoryBlock memoryBlock = - new AtomicLongMemoryBlock("Consensus-Default", null, Runtime.getRuntime().maxMemory() / 10); + new AtomicLongMemoryBlock("Consensus-Default", null, Runtime.getRuntime().totalMemory() / 10); private Double maxMemoryRatioForQueue = 0.6; private IoTConsensusMemoryManager() { diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh index b1f3c687326..d3f849bc745 100755 --- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh +++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh @@ -138,7 +138,10 @@ calculate_memory_sizes() fi off_heap_memory_size_in_mb=`expr $memory_size_in_mb - $on_heap_memory_size_in_mb` + on_heap_memory_size_in_mb_min=`expr $on_heap_memory_size_in_mb / 2` + ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M" + ON_HEAP_MEMORY_MIN="${on_heap_memory_size_in_mb_min}M" OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M" } @@ -309,7 +312,7 @@ else fi IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Diotdb.jmx.local=$JMX_LOCAL" -IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${ON_HEAP_MEMORY}" +IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${ON_HEAP_MEMORY_MIN}" IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${ON_HEAP_MEMORY}" IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}" IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}" @@ -343,6 +346,6 @@ IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+SafepointTimeout" # IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${heap_dump_dir}/datanode_heapdump.hprof" -echo "DataNode on heap memory size = ${ON_HEAP_MEMORY}B, off heap memory size = ${OFF_HEAP_MEMORY}B" +echo "DataNode on max heap memory size = ${ON_HEAP_MEMORY}B, min heap memory size = ${ON_HEAP_MEMORY_MIN}B, off heap memory size = ${OFF_HEAP_MEMORY}B" echo "If you want to change this configuration, please check conf/datanode-env.sh." diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 8a2fe0eef5a..ec1f2c7b795 100755 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2196,35 +2196,35 @@ public class IoTDBDescriptor { } } - long storageEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; - long queryEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; - long schemaEngineMemorySize = Runtime.getRuntime().maxMemory() / 10; - long consensusMemorySize = Runtime.getRuntime().maxMemory() / 10; - long pipeMemorySize = Runtime.getRuntime().maxMemory() / 10; + long storageEngineMemorySize = Runtime.getRuntime().totalMemory() * 3 / 10; + long queryEngineMemorySize = Runtime.getRuntime().totalMemory() * 3 / 10; + long schemaEngineMemorySize = Runtime.getRuntime().totalMemory() / 10; + long consensusMemorySize = Runtime.getRuntime().totalMemory() / 10; + long pipeMemorySize = Runtime.getRuntime().totalMemory() / 10; if (memoryAllocateProportion != null) { String[] proportions = memoryAllocateProportion.split(":"); int proportionSum = 0; for (String proportion : proportions) { proportionSum += Integer.parseInt(proportion.trim()); } - long maxMemoryAvailable = Runtime.getRuntime().maxMemory(); + long memoryAvailable = Runtime.getRuntime().totalMemory(); if (proportionSum != 0) { storageEngineMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum; queryEngineMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum; schemaEngineMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum; consensusMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum; // if pipe proportion is set, use it, otherwise use the default value if (proportions.length >= 6) { pipeMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; } else { pipeMemorySize = - (maxMemoryAvailable + (memoryAvailable - (conf.getStorageEngineMemoryManager().getTotalMemorySizeInBytes() + conf.getQueryEngineMemoryManager().getTotalMemorySizeInBytes() + conf.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes() @@ -2235,7 +2235,7 @@ public class IoTDBDescriptor { } // on heap memory manager MemoryManager onheapMemoryManager = - globalMemoryManager.getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().maxMemory()); + globalMemoryManager.getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().totalMemory()); conf.setOnHeapMemoryManager(onheapMemoryManager); // storage engine memory manager MemoryManager storageEngineMemoryManager = @@ -2449,15 +2449,15 @@ public class IoTDBDescriptor { String queryMemoryAllocateProportion = properties.getProperty("chunk_timeseriesmeta_free_memory_proportion"); - long maxMemoryAvailable = queryEngineMemoryManager.getTotalMemorySizeInBytes(); - - long bloomFilterCacheMemorySize = maxMemoryAvailable / 1001; - long chunkCacheMemorySize = maxMemoryAvailable * 100 / 1001; - long timeSeriesMetaDataCacheMemorySize = maxMemoryAvailable * 200 / 1001; - long coordinatorMemorySize = maxMemoryAvailable * 50 / 1001; - long operatorsMemorySize = maxMemoryAvailable * 200 / 1001; - long dataExchangeMemorySize = maxMemoryAvailable * 200 / 1001; - long timeIndexMemorySize = maxMemoryAvailable * 200 / 1001; + long memoryAvailable = queryEngineMemoryManager.getTotalMemorySizeInBytes(); + + long bloomFilterCacheMemorySize = memoryAvailable / 1001; + long chunkCacheMemorySize = memoryAvailable * 100 / 1001; + long timeSeriesMetaDataCacheMemorySize = memoryAvailable * 200 / 1001; + long coordinatorMemorySize = memoryAvailable * 50 / 1001; + long operatorsMemorySize = memoryAvailable * 200 / 1001; + long dataExchangeMemorySize = memoryAvailable * 200 / 1001; + long timeIndexMemorySize = memoryAvailable * 200 / 1001; if (queryMemoryAllocateProportion != null) { String[] proportions = queryMemoryAllocateProportion.split(":"); int proportionSum = 0; @@ -2467,19 +2467,19 @@ public class IoTDBDescriptor { if (proportionSum != 0) { try { bloomFilterCacheMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum; chunkCacheMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum; timeSeriesMetaDataCacheMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum; coordinatorMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum; operatorsMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; dataExchangeMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[5].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[5].trim()) / proportionSum; timeIndexMemorySize = - maxMemoryAvailable * Integer.parseInt(proportions[6].trim()) / proportionSum; + memoryAvailable * Integer.parseInt(proportions[6].trim()) / proportionSum; } catch (Exception e) { throw new RuntimeException( "Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion" @@ -2699,14 +2699,14 @@ public class IoTDBDescriptor { for (String proportion : proportions) { proportionSum += Integer.parseInt(proportion.trim()); } - float maxMemoryAvailable = conf.getUdfMemoryBudgetInMB(); + float memoryAvailable = conf.getUdfMemoryBudgetInMB(); try { conf.setUdfReaderMemoryBudgetInMB( - maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); + memoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); conf.setUdfTransformerMemoryBudgetInMB( - maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); + memoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); conf.setUdfCollectorMemoryBudgetInMB( - maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum); + memoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum); } catch (Exception e) { throw new RuntimeException( "Each subsection of configuration item udf_reader_transformer_collector_memory_proportion"
