This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch SortBufferSize in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 18cc581b6f1b4c00d2d086a8929949df0d698d14 Author: JackieTien97 <[email protected]> AuthorDate: Tue Jul 22 11:27:14 2025 +0800 Change sort_buffer_size_in_bytes default value --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 28 ++++++++++++++++++---- .../conf/iotdb-system.properties.template | 6 +++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index b751c150585..2493ad221ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -73,6 +73,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.CONSENSUS_FOLDER_NAME; import static org.apache.iotdb.commons.conf.IoTDBConstant.DELETION_FOLDER_NAME; import static org.apache.iotdb.commons.conf.IoTDBConstant.OBJECT_STORAGE_DIR; import static org.apache.iotdb.commons.conf.IoTDBConstant.PIPE_FOLDER_NAME; +import static org.apache.iotdb.db.conf.IoTDBDescriptor.calculateDefaultSortBufferSize; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class IoTDBConfig { @@ -415,7 +416,8 @@ public class IoTDBConfig { private volatile boolean enableAutoRepairCompaction = true; /** The buffer for sort operation */ - private long sortBufferSize = 1024 * 1024L; + private long sortBufferSize = + calculateDefaultSortBufferSize(IoTDBDescriptor.getInstance().getMemoryConfig()); /** * The strategy of inner space compaction task. There are just one inner space compaction strategy diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 34501ffff69..7a6bfc8aa58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1027,10 +1027,7 @@ public class IoTDBDescriptor { properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable())))); // The buffer for sort operator to calculate - conf.setSortBufferSize( - Long.parseLong( - properties.getProperty( - "sort_buffer_size_in_bytes", Long.toString(conf.getSortBufferSize())))); + loadSortBuffer(properties); // tmp filePath for sort operator conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir())); @@ -1093,6 +1090,26 @@ public class IoTDBDescriptor { loadTrustedUriPattern(properties); } + private void loadSortBuffer(TrimProperties properties) { + long defaultValue = calculateDefaultSortBufferSize(memoryConfig); + long sortBufferSize = + Long.parseLong( + properties.getProperty("sort_buffer_size_in_bytes", Long.toString(defaultValue))); + if (sortBufferSize <= 0) { + sortBufferSize = defaultValue; + } + // The buffer for sort operator to calculate + conf.setSortBufferSize(sortBufferSize); + } + + public static long calculateDefaultSortBufferSize(DataNodeMemoryConfig memoryConfig) { + return Math.min( + 32 * 1024 * 1024L, + memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes() + / memoryConfig.getQueryThreadCount() + / 2); + } + private void reloadConsensusProps(TrimProperties properties) throws IOException { loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); @@ -2064,6 +2081,9 @@ public class IoTDBDescriptor { properties.getProperty( "tvlist_sort_threshold", ConfigurationFileUtils.getConfigurationDefaultValue("tvlist_sort_threshold")))); + + // sort_buffer_size_in_bytes + loadSortBuffer(properties); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 28af70751ba..67811dacadb 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1070,9 +1070,11 @@ mode_map_size_threshold=10000 batch_size=100000 # The memory for external sort in sort operator, when the data size is smaller than sort_buffer_size_in_bytes, the sort operator will use in-memory sort. -# effectiveMode: restart +# if sort_buffer_size_in_bytes <= 0, default value will be used, default value = min(32MB, memory for query operators / query_thread_count / 2) +# if sort_buffer_size_in_bytes > 0, the specified value will be used. +# effectiveMode: hot_reload # Datatype: long -sort_buffer_size_in_bytes=1048576 +sort_buffer_size_in_bytes=0 # The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged. # effectiveMode: hot_reload
