This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch Optimize_Storage_Engine_allocate in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a413de0bc4c049076880518ffbc07b3262df32df Author: HTHou <[email protected]> AuthorDate: Thu Jun 8 12:29:51 2023 +0800 Fix storage Engine memory allocate error --- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 74 +++++++++++++--------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7d3093d1ba0..91751f44786 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1699,13 +1699,12 @@ public class IoTDBDescriptor { conf.setAllocateMemoryForConsensus( maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum); } + logger.info("initial allocateMemoryForRead = {}", conf.getAllocateMemoryForRead()); + logger.info("initial allocateMemoryForWrite = {}", conf.getAllocateMemoryForStorageEngine()); + logger.info("initial allocateMemoryForSchema = {}", conf.getAllocateMemoryForSchema()); + logger.info("initial allocateMemoryForConsensus = {}", conf.getAllocateMemoryForConsensus()); } - logger.info("initial allocateMemoryForRead = {}", conf.getAllocateMemoryForRead()); - logger.info("initial allocateMemoryForWrite = {}", conf.getAllocateMemoryForStorageEngine()); - logger.info("initial allocateMemoryForSchema = {}", conf.getAllocateMemoryForSchema()); - logger.info("initial allocateMemoryForConsensus = {}", conf.getAllocateMemoryForConsensus()); - initSchemaMemoryAllocate(properties); initStorageEngineAllocate(properties); @@ -1767,36 +1766,53 @@ public class IoTDBDescriptor { } private void initStorageEngineAllocate(Properties properties) { - String allocationRatio = properties.getProperty("storage_engine_memory_proportion", "8:2"); - String[] proportions = allocationRatio.split(":"); - int proportionForWrite = Integer.parseInt(proportions[0].trim()); - int proportionForCompaction = Integer.parseInt(proportions[1].trim()); + long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine(); - double writeProportion = - ((double) (proportionForWrite) / (double) (proportionForCompaction + proportionForWrite)); + int proportionSum = 10; + int writeProportion = 8; + int compactionProportion = 2; + int writeProportionSum = 20; + int memTableProportion = 19; + int timePartitionInfo = 1; + + String storageMemoryAllocatePortion = + properties.getProperty("storage_engine_memory_proportion"); + if (storageMemoryAllocatePortion != null) { + String[] proportions = storageMemoryAllocatePortion.split(":"); + int loadedProportionSum = 0; + for (String proportion : proportions) { + loadedProportionSum += Integer.parseInt(proportion.trim()); + } - String allocationRatioForWrite = properties.getProperty("write_memory_proportion", "19:1"); - proportions = allocationRatioForWrite.split(":"); - int proportionForMemTable = Integer.parseInt(proportions[0].trim()); - int proportionForTimePartitionInfo = Integer.parseInt(proportions[1].trim()); + if (loadedProportionSum != 0) { + proportionSum = loadedProportionSum; + writeProportion = Integer.parseInt(proportions[0].trim()); + compactionProportion = Integer.parseInt(proportions[1].trim()); + } + } - double memtableProportionForWrite = - ((double) (proportionForMemTable) - / (double) (proportionForMemTable + proportionForTimePartitionInfo)); + conf.setCompactionProportion((double) compactionProportion / (double) proportionSum); + String allocationRatioForWrite = properties.getProperty("write_memory_proportion"); + if (allocationRatioForWrite != null) { + String[] proportions = allocationRatioForWrite.split(":"); + int loadedProportionSum = 0; + for (String proportion : proportions) { + loadedProportionSum += Integer.parseInt(proportion.trim()); + } - double timePartitionInfoForWrite = - ((double) (proportionForTimePartitionInfo) - / (double) (proportionForMemTable + proportionForTimePartitionInfo)); - conf.setWriteProportionForMemtable(writeProportion * memtableProportionForWrite); + if (loadedProportionSum != 0) { + writeProportionSum = loadedProportionSum; + memTableProportion = Integer.parseInt(proportions[0].trim()); + timePartitionInfo = Integer.parseInt(proportions[1].trim()); + } + } + double memtableProportionForWrite = ((double) memTableProportion / (double) writeProportionSum); + double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum); + double proportionForWrite = ((double) (writeProportion) / (double) proportionSum); + conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite); conf.setAllocateMemoryForTimePartitionInfo( - (long) - ((writeProportion * timePartitionInfoForWrite) - * conf.getAllocateMemoryForStorageEngine())); - - conf.setCompactionProportion( - ((double) (proportionForCompaction) - / (double) (proportionForCompaction + proportionForWrite))); + (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal)); } private void initSchemaMemoryAllocate(Properties properties) {
