This is an automated email from the ASF dual-hosted git repository. Wei-hao-Li pushed a commit to branch mppEx in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 95c4176f640d30bb4e1b01762a1b8bb7e733a111 Author: Weihao Li <[email protected]> AuthorDate: Tue Sep 15 12:59:16 2026 +0800 config hotReload Signed-off-by: Weihao Li <[email protected]> --- .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 6 ++++ .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 6 ++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 ++-- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 35 ++++++++++++++++++---- .../conf/iotdb-system.properties.template | 10 ++++--- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 12f4e374ca7..0cdcce0121e 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1488,5 +1488,11 @@ public final class DataNodeMiscMessages { public static final String LOG_TABLE_QUERY_DEVICE_ENTRY_BATCH_SIZE_IN_BYTES_ARG_EXCEEDS_DN_THRIFT_MAX_FRAME_SIZE_ARG_USING_ARG_AS_THE_EFFECTIVE_VALUE_2AE1BEDA = "table_query_device_entry_batch_size_in_bytes (%d) exceeds the maximum RPC payload (dn_thrift_max_frame_size %d minus 1024 bytes); using %d as the effective value"; + public static final String + LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_NOT_POSITIVE_USING_DEFAULT_VALUE_ARG_1AA821B2 = + "mpp_data_exchange_max_payload_size_in_bytes (%d) is not positive, using default value %d"; + public static final String + LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_EXCEEDS_MAXIMUM_ALLOWED_VALUE_ARG_USING_ARG_D9BF0BBC = + "mpp_data_exchange_max_payload_size_in_bytes (%d) exceeds the maximum allowed value %d, using %d"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index d809fea1992..155ca9f0c4d 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1468,5 +1468,11 @@ public final class DataNodeMiscMessages { public static final String LOG_TABLE_QUERY_DEVICE_ENTRY_BATCH_SIZE_IN_BYTES_ARG_EXCEEDS_DN_THRIFT_MAX_FRAME_SIZE_ARG_USING_ARG_AS_THE_EFFECTIVE_VALUE_2AE1BEDA = "table_query_device_entry_batch_size_in_bytes（%d）超过最大 RPC payload（dn_thrift_max_frame_size %d 减去 1024 字节），将使用 %d 作为生效值"; + public static final String + LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_NOT_POSITIVE_USING_DEFAULT_VALUE_ARG_1AA821B2 = + "mpp_data_exchange_max_payload_size_in_bytes（%d）不是正数，将使用默认值 %d"; + public static final String + LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_EXCEEDS_MAXIMUM_ALLOWED_VALUE_ARG_USING_ARG_D9BF0BBC = + "mpp_data_exchange_max_payload_size_in_bytes（%d）超过允许的最大值 %d，将使用 %d"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index ff141a8ed85..6d169eda42c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -956,7 +956,7 @@ public class IoTDBConfig { /** Core pool size of mpp data exchange. */ private int mppDataExchangeCorePoolSize = 10; - private int mppDataExchangeMaxPayloadSizeInBytes = 8 * 1024 * 1024; + private int mppDataExchangeMaxPayloadSizeInBytes = 4 * 1024 * 1024; /** Max pool size of mpp data exchange. */ private int mppDataExchangeMaxPoolSize = 10; @@ -3428,8 +3428,7 @@ public class IoTDBConfig { } public void setMppDataExchangeMaxPayloadSizeInBytes(int mppDataExchangeMaxPayloadSizeInBytes) { - this.mppDataExchangeMaxPayloadSizeInBytes = - Math.max(1, Math.min(mppDataExchangeMaxPayloadSizeInBytes, thriftMaxFrameSize - 1024)); + this.mppDataExchangeMaxPayloadSizeInBytes = mppDataExchangeMaxPayloadSizeInBytes; } public int getConnectionTimeoutInMS() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 8c3efb84b83..83cdf3cac10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2268,6 +2268,7 @@ public class IoTDBDescriptor { "enable_topk_runtime_filter")))); loadTableQueryDeviceEntryBatchSize(properties); + loadMppDataExchangeMaxPayloadSize(properties); // update wal config long prevDeleteWalFilesPeriodInMs = conf.getDeleteWalFilesPeriodInMs(); @@ -3070,11 +3071,7 @@ public class IoTDBDescriptor { properties.getProperty( "mpp_data_exchange_keep_alive_time_in_ms", Integer.toString(conf.getMppDataExchangeKeepAliveTimeInMs())))); - conf.setMppDataExchangeMaxPayloadSizeInBytes( - Integer.parseInt( - properties.getProperty( - "mpp_data_exchange_max_payload_size_in_bytes", - Integer.toString(conf.getMppDataExchangeMaxPayloadSizeInBytes())))); + loadMppDataExchangeMaxPayloadSize(properties); conf.setPartitionCacheSize( Integer.parseInt( @@ -3088,6 +3085,34 @@ public class IoTDBDescriptor { Integer.toString(commonConfig.getDriverTaskExecutionTimeSliceInMs())))); } + private void loadMppDataExchangeMaxPayloadSize(TrimProperties properties) { + int configuredSize = + Integer.parseInt( + properties.getProperty( + "mpp_data_exchange_max_payload_size_in_bytes", + Integer.toString(conf.getMppDataExchangeMaxPayloadSizeInBytes()))); + int defaultSize = 4 * 1024 * 1024; + if (configuredSize <= 0) { + LOGGER.warn( + DataNodeMiscMessages + .LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_NOT_POSITIVE_USING_DEFAULT_VALUE_ARG_1AA821B2, + configuredSize, + defaultSize); + configuredSize = defaultSize; + } + int maxAllowedSize = Math.max(1, conf.getThriftMaxFrameSize() - 1024); + if (configuredSize > maxAllowedSize) { + LOGGER.warn( + DataNodeMiscMessages + .LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_EXCEEDS_MAXIMUM_ALLOWED_VALUE_ARG_USING_ARG_D9BF0BBC, + configuredSize, + maxAllowedSize, + maxAllowedSize); + configuredSize = maxAllowedSize; + } + conf.setMppDataExchangeMaxPayloadSizeInBytes(configuredSize); + } + /** Get default encode algorithm by data type */ public TSEncoding getDefaultEncodingByType(TSDataType dataType) { switch (dataType) { diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index f7665c30098..2a51650c0cc 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -71,7 +71,7 @@ cn_consensus_port=10720 # Used for connection of IoTDB native clients(Session) # Could set 127.0.0.1(for local test), ipv4/ipv6 address, or hostname. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: String dn_rpc_address=127.0.0.1 @@ -1194,10 +1194,12 @@ mpp_data_exchange_max_pool_size=10 # Datatype: int mpp_data_exchange_keep_alive_time_in_ms=1000 -# The maximum payload size of one MPP data exchange RPC response -# effectiveMode: restart +# The maximum payload size of one MPP data exchange RPC response. +# <=0 use default value +# The effective value is capped by dn_thrift_max_frame_size minus 1024 bytes reserved for the RPC response envelope. +# effectiveMode: hot_reload # Datatype: int, Unit: byte -mpp_data_exchange_max_payload_size_in_bytes=8388608 +mpp_data_exchange_max_payload_size_in_bytes=4194304 # The max execution time of a DriverTask # effectiveMode: restart
