This is an automated email from the ASF dual-hosted git repository. jt2594838 pushed a commit to branch add_rpc_memory_control in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4b4003c64aece29bb5c570874811c90f607ecb09 Author: Tian Jiang <[email protected]> AuthorDate: Wed Jun 10 18:01:35 2026 +0800 ver3 --- .../iotdb/it/env/cluster/config/MppCommonConfig.java | 9 +++++++++ .../it/env/cluster/config/MppSharedCommonConfig.java | 8 ++++++++ .../iotdb/it/env/remote/config/RemoteCommonConfig.java | 6 ++++++ .../java/org/apache/iotdb/itbase/env/CommonConfig.java | 2 ++ .../resources/conf/iotdb-system.properties.template | 5 +++++ .../java/org/apache/iotdb/commons/conf/CommonConfig.java | 16 ++++++++++++++++ .../org/apache/iotdb/commons/conf/CommonDescriptor.java | 5 +++++ .../org/apache/iotdb/commons/memory/MemoryConfig.java | 9 ++++++--- 8 files changed, 57 insertions(+), 3 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 4852b9d116e..6d8192aa85e 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -417,6 +417,15 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig { return this; } + @Override + public CommonConfig setAutoResizingBufferMemoryProportion( + double autoResizingBufferMemoryProportion) { + setProperty( + "auto_resizing_buffer_memory_proportion", + String.valueOf(autoResizingBufferMemoryProportion)); + return this; + } + @Override public CommonConfig setQuotaEnable(boolean quotaEnable) { setProperty("quota_enable", String.valueOf(quotaEnable)); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 582c9a049e4..2cf435d49ea 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -423,6 +423,14 @@ public class MppSharedCommonConfig implements CommonConfig { return this; } + @Override + public CommonConfig setAutoResizingBufferMemoryProportion( + double autoResizingBufferMemoryProportion) { + dnConfig.setAutoResizingBufferMemoryProportion(autoResizingBufferMemoryProportion); + cnConfig.setAutoResizingBufferMemoryProportion(autoResizingBufferMemoryProportion); + return this; + } + @Override public CommonConfig setQuotaEnable(boolean quotaEnable) { dnConfig.setQuotaEnable(quotaEnable); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index 48c157e957b..f554c463b9f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -299,6 +299,12 @@ public class RemoteCommonConfig implements CommonConfig { return this; } + @Override + public CommonConfig setAutoResizingBufferMemoryProportion( + double autoResizingBufferMemoryProportion) { + return this; + } + @Override public CommonConfig setQuotaEnable(boolean quotaEnable) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index dc21234e2ba..85cc0afc1a4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -134,6 +134,8 @@ public interface CommonConfig { CommonConfig setWriteMemoryProportion(String writeMemoryProportion); + CommonConfig setAutoResizingBufferMemoryProportion(double autoResizingBufferMemoryProportion); + CommonConfig setClusterTimeseriesLimitThreshold(long clusterTimeseriesLimitThreshold); CommonConfig setClusterDeviceLimitThreshold(long clusterDeviceLimitThreshold); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 531cbac91a8..20156be9315 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -786,6 +786,11 @@ partition_table_recover_max_read_megabytes_per_second=10 # effectiveMode: restart datanode_memory_proportion=3:3:1:1:1:1 +# The proportion of GlobalMemoryManager used by AutoResizingBuffer. +# effectiveMode: restart +# Datatype: double +auto_resizing_buffer_memory_proportion=0.01 + # Schema Memory Allocation Ratio: SchemaRegion, SchemaCache, and PartitionCache. # The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:2:1 # effectiveMode: restart diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 6a8956e423b..b2cf6775106 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -416,6 +416,8 @@ public class CommonConfig { private long seriesLimitThreshold = -1; private long deviceLimitThreshold = -1; + private double autoResizingBufferMemoryProportion = 0.01; + private boolean enableBinaryAllocator = true; private int arenaNum = 4; @@ -2651,6 +2653,20 @@ public class CommonConfig { this.deviceLimitThreshold = deviceLimitThreshold; } + public double getAutoResizingBufferMemoryProportion() { + return autoResizingBufferMemoryProportion; + } + + public void setAutoResizingBufferMemoryProportion(double autoResizingBufferMemoryProportion) { + if (autoResizingBufferMemoryProportion <= 0 || autoResizingBufferMemoryProportion > 1) { + logger.warn( + "autoResizingBufferMemoryProportion should be in (0, 1], but was {}", + autoResizingBufferMemoryProportion); + return; + } + this.autoResizingBufferMemoryProportion = autoResizingBufferMemoryProportion; + } + public long getStartUpNanosecond() { return startUpNanosecond; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 5cd954a09f7..e893a24e6f2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -300,6 +300,11 @@ public class CommonDescriptor { properties.getProperty( "cluster_device_limit_threshold", String.valueOf(config.getDeviceLimitThreshold())))); + config.setAutoResizingBufferMemoryProportion( + Double.parseDouble( + properties.getProperty( + "auto_resizing_buffer_memory_proportion", + String.valueOf(config.getAutoResizingBufferMemoryProportion())))); config.setPathLogMaxSize( Integer.parseInt( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java index cde5882ba59..2c2b22a172d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java @@ -19,13 +19,13 @@ package org.apache.iotdb.commons.memory; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.rpc.AutoResizingBufferMemoryControl; import org.apache.iotdb.rpc.AutoResizingBufferMemoryManager; public class MemoryConfig { private static final String AUTO_RESIZING_BUFFER_MEMORY_MANAGER_NAME = "AutoResizingBuffer"; private static final String AUTO_RESIZING_BUFFER_MEMORY_BLOCK_NAME = "AutoResizingBufferBlock"; - private static final int AUTO_RESIZING_BUFFER_MEMORY_RATIO_DENOMINATOR = 100; private final MemoryManager globalMemoryManager = new MemoryManager("GlobalMemoryManager", null, Runtime.getRuntime().totalMemory()); @@ -70,8 +70,11 @@ public class MemoryConfig { if (autoResizingBufferMemoryBlock == null || autoResizingBufferMemoryBlock.isReleased()) { long autoResizingBufferMemorySize = - globalMemoryManager.getTotalMemorySizeInBytes() - / AUTO_RESIZING_BUFFER_MEMORY_RATIO_DENOMINATOR; + (long) + (globalMemoryManager.getTotalMemorySizeInBytes() + * CommonDescriptor.getInstance() + .getConfig() + .getAutoResizingBufferMemoryProportion()); MemoryManager autoResizingBufferMemoryManager = globalMemoryManager.getOrCreateMemoryManager( AUTO_RESIZING_BUFFER_MEMORY_MANAGER_NAME, autoResizingBufferMemorySize, true);
