This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2459e5211ce Change sort_buffer_size_in_bytes default value
2459e5211ce is described below
commit 2459e5211ce7ded5a355ff6d96abbdcda10700f9
Author: Jackie Tien <[email protected]>
AuthorDate: Tue Jul 29 14:16:59 2025 +0800
Change sort_buffer_size_in_bytes default value
---
.../relational/it/db/it/IoTDBWindowFunctionIT.java | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 28 ++++++++++++++++++----
.../conf/iotdb-system.properties.template | 6 +++--
4 files changed, 30 insertions(+), 7 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java
index d99de124f24..c3ddf198366 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java
@@ -107,6 +107,7 @@ public class IoTDBWindowFunctionIT {
@BeforeClass
public static void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(1024 *
1024);
EnvFactory.getEnv().initClusterEnvironment();
insertData();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 30c343ba358..75dac521f2f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -415,7 +415,7 @@ public class IoTDBConfig {
private volatile boolean enableAutoRepairCompaction = true;
/** The buffer for sort operation */
- private long sortBufferSize = 1024 * 1024L;
+ private long sortBufferSize = 32 * 1024 * 1024L;
/**
* The strategy of inner space compaction task. There are just one inner
space compaction strategy
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 34501ffff69..7a6bfc8aa58 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1027,10 +1027,7 @@ public class IoTDBDescriptor {
properties.getProperty("quota_enable",
String.valueOf(conf.isQuotaEnable()))));
// The buffer for sort operator to calculate
- conf.setSortBufferSize(
- Long.parseLong(
- properties.getProperty(
- "sort_buffer_size_in_bytes",
Long.toString(conf.getSortBufferSize()))));
+ loadSortBuffer(properties);
// tmp filePath for sort operator
conf.setSortTmpDir(properties.getProperty("sort_tmp_dir",
conf.getSortTmpDir()));
@@ -1093,6 +1090,26 @@ public class IoTDBDescriptor {
loadTrustedUriPattern(properties);
}
+ private void loadSortBuffer(TrimProperties properties) {
+ long defaultValue = calculateDefaultSortBufferSize(memoryConfig);
+ long sortBufferSize =
+ Long.parseLong(
+ properties.getProperty("sort_buffer_size_in_bytes",
Long.toString(defaultValue)));
+ if (sortBufferSize <= 0) {
+ sortBufferSize = defaultValue;
+ }
+ // The buffer for sort operator to calculate
+ conf.setSortBufferSize(sortBufferSize);
+ }
+
+ public static long calculateDefaultSortBufferSize(DataNodeMemoryConfig
memoryConfig) {
+ return Math.min(
+ 32 * 1024 * 1024L,
+ memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes()
+ / memoryConfig.getQueryThreadCount()
+ / 2);
+ }
+
private void reloadConsensusProps(TrimProperties properties) throws
IOException {
loadIoTConsensusProps(properties);
loadIoTConsensusV2Props(properties);
@@ -2064,6 +2081,9 @@ public class IoTDBDescriptor {
properties.getProperty(
"tvlist_sort_threshold",
ConfigurationFileUtils.getConfigurationDefaultValue("tvlist_sort_threshold"))));
+
+ // sort_buffer_size_in_bytes
+ loadSortBuffer(properties);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 28af70751ba..67811dacadb 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1070,9 +1070,11 @@ mode_map_size_threshold=10000
batch_size=100000
# The memory for external sort in sort operator, when the data size is smaller
than sort_buffer_size_in_bytes, the sort operator will use in-memory sort.
-# effectiveMode: restart
+# if sort_buffer_size_in_bytes <= 0, default value will be used, default value
= min(32MB, memory for query operators / query_thread_count / 2)
+# if sort_buffer_size_in_bytes > 0, the specified value will be used.
+# effectiveMode: hot_reload
# Datatype: long
-sort_buffer_size_in_bytes=1048576
+sort_buffer_size_in_bytes=0
# The threshold of operator count in the result set of EXPLAIN ANALYZE, if the
number of operator in the result set is larger than this threshold, operator
will be merged.
# effectiveMode: hot_reload
