This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e615946bab0 Fix startup with invalid storage memory proportions
(#18979)
e615946bab0 is described below
commit e615946bab03616d8b960e5752fdf2beb52267e2
Author: Hongzhi Gao <[email protected]>
AuthorDate: Fri Oct 23 11:29:35 2026 +0800
Fix startup with invalid storage memory proportions (#18979)
---
.../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 102 ++++++++++-----------
.../iotdb/db/conf/DataNodeMemoryConfigTest.java | 98 ++++++++++++++++++++
2 files changed, 147 insertions(+), 53 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
index 12ac2293d3d..6f93f057ccf 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
@@ -399,59 +399,30 @@ public class DataNodeMemoryConfig {
Double.toString(getBufferedArraysMemoryProportion()))));
long storageMemoryTotal =
storageEngineMemoryManager.getTotalMemorySizeInBytes();
- String valueOfStorageEngineMemoryProportion =
- properties.getProperty("storage_engine_memory_proportion");
- long writeMemorySize = storageMemoryTotal * 8 / 10;
- long compactionMemorySize = storageMemoryTotal * 2 / 10;
- long memtableMemorySize = writeMemorySize * 19 / 20;
- long timePartitionInfoMemorySize = writeMemorySize / 20;
- if (valueOfStorageEngineMemoryProportion != null) {
- String[] storageProportionArray =
valueOfStorageEngineMemoryProportion.split(":");
- int storageEngineMemoryProportion = 0;
- for (String proportion : storageProportionArray) {
- int proportionValue = Integer.parseInt(proportion.trim());
- if (proportionValue <= 0) {
- LOGGER.warn(
- DataNodeMiscMessages
-
.MISC_LOG_THE_VALUE_OF_STORAGE_ENGINE_MEMORY_PROPORTION_IS_ILLEGAL_22CA9433);
- return;
- }
- storageEngineMemoryProportion += proportionValue;
- }
- writeMemorySize =
- storageMemoryTotal
- * Integer.parseInt(storageProportionArray[0].trim())
- / storageEngineMemoryProportion;
- compactionMemorySize =
- storageMemoryTotal
- * Integer.parseInt(storageProportionArray[1].trim())
- / storageEngineMemoryProportion;
-
- String valueOfWriteMemoryProportion =
properties.getProperty("write_memory_proportion");
- if (valueOfWriteMemoryProportion != null) {
- String[] writeProportionArray =
valueOfWriteMemoryProportion.split(":");
- int writeMemoryProportion = 0;
- for (String proportion : writeProportionArray) {
- int proportionValue = Integer.parseInt(proportion.trim());
- writeMemoryProportion += proportionValue;
- if (proportionValue <= 0) {
- LOGGER.warn(
- DataNodeMiscMessages
-
.MISC_LOG_THE_VALUE_OF_WRITE_MEMORY_PROPORTION_IS_ILLEGAL_USE_DEFAULT_EE4FA112);
- return;
- }
- }
-
- memtableMemorySize =
- writeMemorySize
- * Integer.parseInt(writeProportionArray[0].trim())
- / writeMemoryProportion;
- timePartitionInfoMemorySize =
- writeMemorySize
- * Integer.parseInt(writeProportionArray[1].trim())
- / writeMemoryProportion;
- }
- }
+ int[] storageProportions =
+ resolveStorageMemoryProportions(
+ properties.getProperty("storage_engine_memory_proportion"),
+ 8,
+ 2,
+ DataNodeMiscMessages
+
.MISC_LOG_THE_VALUE_OF_STORAGE_ENGINE_MEMORY_PROPORTION_IS_ILLEGAL_22CA9433);
+ long storageProportionSum = (long) storageProportions[0] +
storageProportions[1];
+ long writeMemorySize =
+ calculateMemoryShare(storageMemoryTotal, storageProportions[0],
storageProportionSum);
+ long compactionMemorySize =
+ calculateMemoryShare(storageMemoryTotal, storageProportions[1],
storageProportionSum);
+ int[] writeProportions =
+ resolveStorageMemoryProportions(
+ properties.getProperty("write_memory_proportion"),
+ 19,
+ 1,
+ DataNodeMiscMessages
+
.MISC_LOG_THE_VALUE_OF_WRITE_MEMORY_PROPORTION_IS_ILLEGAL_USE_DEFAULT_EE4FA112);
+ long writeProportionSum = (long) writeProportions[0] + writeProportions[1];
+ long memtableMemorySize =
+ calculateMemoryShare(writeMemorySize, writeProportions[0],
writeProportionSum);
+ long timePartitionInfoMemorySize =
+ calculateMemoryShare(writeMemorySize, writeProportions[1],
writeProportionSum);
writeMemoryManager =
storageEngineMemoryManager.getOrCreateMemoryManager("Write",
writeMemorySize);
initCompactionMemoryManager(compactionMemorySize);
@@ -473,6 +444,31 @@ public class DataNodeMemoryConfig {
memtableMemoryManager.getOrCreateMemoryManager("WalBufferQueue",
walBufferQueueMemorySize);
}
+ private static int[] resolveStorageMemoryProportions(
+ String configuredProportions, int defaultFirst, int defaultSecond,
String warningMessage) {
+ if (configuredProportions != null) {
+ String[] proportions = configuredProportions.split(":", -1);
+ if (proportions.length == 2) {
+ try {
+ int first = Integer.parseInt(proportions[0].trim());
+ int second = Integer.parseInt(proportions[1].trim());
+ if (first > 0 && second > 0) {
+ return new int[] {first, second};
+ }
+ } catch (NumberFormatException e) {
+ // Malformed or out-of-range configuration values use the default
proportions.
+ }
+ }
+ LOGGER.warn(warningMessage);
+ }
+ return new int[] {defaultFirst, defaultSecond};
+ }
+
+ private static long calculateMemoryShare(long total, int proportion, long
proportionSum) {
+ // Divide first to avoid overflowing when a valid proportion is close to
Integer.MAX_VALUE.
+ return total / proportionSum * proportion + total % proportionSum *
proportion / proportionSum;
+ }
+
private void initCompactionMemoryManager(long compactionMemorySize) {
long fixedMemoryCost =
AbstractCompactionEstimator.allocateMemoryCostForFileInfoCache(compactionMemorySize);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java
index 6aeeb22f2f6..af73f0a78cd 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java
@@ -26,7 +26,9 @@ import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
import org.junit.Test;
+import java.lang.reflect.Field;
import java.lang.reflect.Method;
+import java.math.BigInteger;
import java.util.Arrays;
import static org.junit.Assert.assertArrayEquals;
@@ -178,6 +180,102 @@ public class DataNodeMemoryConfigTest {
DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(properties));
}
+ @Test
+ public void testInvalidStorageMemoryProportionsUseDefaults() throws
ReflectiveOperationException {
+ for (String invalid :
+ new String[] {
+ "0:10", "10:0", "-1:2", "10", "a:b", "", ":2", "8:", "8:2:",
"8:2:1", "2147483648:1"
+ }) {
+ TrimProperties properties = new TrimProperties();
+ properties.setProperty("storage_engine_memory_proportion", invalid);
+ properties.setProperty("write_memory_proportion", "3:1");
+ assertStorageMemoryBudgets(properties, 800_000, 600_000, 200_000);
+
+ properties.setProperty("storage_engine_memory_proportion", "1:1");
+ properties.setProperty("write_memory_proportion", invalid);
+ assertStorageMemoryBudgets(properties, 500_000, 475_000, 25_000);
+ }
+ }
+
+ @Test
+ public void testStorageMemoryProportionsAreResolvedIndependently()
+ throws ReflectiveOperationException {
+ TrimProperties properties = new TrimProperties();
+ assertStorageMemoryBudgets(properties, 800_000, 760_000, 40_000);
+ properties.setProperty("write_memory_proportion", " 3 : 1 ");
+ assertStorageMemoryBudgets(properties, 800_000, 600_000, 200_000);
+ properties.remove("write_memory_proportion");
+ properties.setProperty("storage_engine_memory_proportion", " 1 : 1 ");
+ assertStorageMemoryBudgets(properties, 500_000, 475_000, 25_000);
+ properties.setProperty("storage_engine_memory_proportion", "0:10");
+ properties.setProperty("write_memory_proportion", "0:1");
+ assertStorageMemoryBudgets(properties, 800_000, 760_000, 40_000);
+ }
+
+ @Test
+ public void testLargeStorageMemoryProportionsDoNotOverflow() throws
ReflectiveOperationException {
+ TrimProperties properties = new TrimProperties();
+ properties.setProperty("storage_engine_memory_proportion",
"2147483647:2147483647");
+ properties.setProperty("write_memory_proportion", "2147483647:2147483647");
+ DataNodeMemoryConfig memoryConfig =
initializeStorageEngineMemory(properties, 32L << 30);
+ assertEquals(16L << 30,
memoryConfig.getWriteMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(8L << 30,
memoryConfig.getMemtableMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ 8L << 30,
memoryConfig.getTimePartitionInfoMemoryManager().getTotalMemorySizeInBytes());
+
+ properties.setProperty("storage_engine_memory_proportion", "2147483646:1");
+ properties.setProperty("write_memory_proportion", "2147483646:1");
+ memoryConfig = initializeStorageEngineMemory(properties, 32L << 30);
+ BigInteger proportion = BigInteger.valueOf(Integer.MAX_VALUE - 1L);
+ BigInteger sum = proportion.add(BigInteger.ONE);
+ BigInteger expectedWrite = BigInteger.valueOf(32L <<
30).multiply(proportion).divide(sum);
+ assertEquals(
+ expectedWrite.longValueExact(),
+ memoryConfig.getWriteMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ expectedWrite.multiply(proportion).divide(sum).longValueExact(),
+ memoryConfig.getMemtableMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ expectedWrite.divide(sum).longValueExact(),
+
memoryConfig.getTimePartitionInfoMemoryManager().getTotalMemorySizeInBytes());
+ }
+
+ private void assertStorageMemoryBudgets(
+ TrimProperties properties, long write, long memtable, long timePartition)
+ throws ReflectiveOperationException {
+ DataNodeMemoryConfig memoryConfig =
initializeStorageEngineMemory(properties, 1_000_000L);
+ assertEquals(write,
memoryConfig.getWriteMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(memtable,
memoryConfig.getMemtableMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ timePartition,
+
memoryConfig.getTimePartitionInfoMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ 1_000_000L - write,
memoryConfig.getCompactionMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ (long) (memtable * memoryConfig.getDevicePathCacheProportion()),
+
memoryConfig.getDevicePathCacheMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ (long) (memtable * memoryConfig.getWalBufferQueueProportion()),
+
memoryConfig.getWalBufferQueueMemoryManager().getTotalMemorySizeInBytes());
+ assertEquals(
+ 600_000L,
memoryConfig.getBufferedArraysMemoryManager().getTotalMemorySizeInBytes());
+ }
+
+ private DataNodeMemoryConfig initializeStorageEngineMemory(TrimProperties
properties, long total)
+ throws ReflectiveOperationException {
+ DataNodeMemoryConfig memoryConfig = new DataNodeMemoryConfig();
+ MemoryManager storageMemoryManager = new MemoryManager(total);
+ Field storageField =
DataNodeMemoryConfig.class.getDeclaredField("storageEngineMemoryManager");
+ storageField.setAccessible(true);
+ storageField.set(memoryConfig, storageMemoryManager);
+ Method initialize =
+ DataNodeMemoryConfig.class.getDeclaredMethod(
+ "initStorageEngineAllocate", MemoryManager.class,
TrimProperties.class);
+ initialize.setAccessible(true);
+ initialize.invoke(memoryConfig, storageMemoryManager, properties);
+ return memoryConfig;
+ }
+
private DataNodeMemoryConfig initializeQueryEngineMemory(TrimProperties
properties)
throws ReflectiveOperationException {
final DataNodeMemoryConfig memoryConfig = new DataNodeMemoryConfig();
