This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/FixIntoMem in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cffbdab37b9788495a26c9f42c3b09412bf7bd2e Author: Minghui Liu <[email protected]> AuthorDate: Fri Apr 7 10:45:20 2023 +0800 add config `into_operation_buffer_size_in_byte` --- docs/UserGuide/Reference/Common-Config-Manual.md | 10 ++++++++++ docs/zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++ .../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +++++++++++++-- .../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 ++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md index b7975023fe..403e39654c 100644 --- a/docs/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/UserGuide/Reference/Common-Config-Manual.md @@ -1392,6 +1392,16 @@ Different configuration parameters take effect in the following three ways: ### SELECT-INTO +* into\_operation\_buffer\_size\_in\_byte + +| Name | into\_operation\_buffer\_size\_in\_byte | +| :---------: | :---------------------------------------------------------------------------------------------------------------------------------- | +| Description | When the select-into statement is executed, the maximum memory occupied by the data to be written (unit: Byte) | +| Type | int64 | +| Default | 100MB | +| Effective | hot-load | + + * select\_into\_insert\_tablet\_plan\_row\_limit | Name | select\_into\_insert\_tablet\_plan\_row\_limit | diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md index 47ffa15a45..f9c5dc0869 100644 --- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md @@ -1430,6 +1430,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。 #### SELECT-INTO配置 +* into\_operation\_buffer\_size\_in\_byte + +| 名字 | into\_operation\_buffer\_size\_in\_byte | +| :----------: | :-------------------------------------------------------------------- | +| 描述 | 执行 select-into 语句时，待写入数据占用的最大内存（单位：Byte） | +| 类型 | int64 | +| 默认值 | 100MB | +| 改后生效方式 | 热加载 | + * select\_into\_insert\_tablet\_plan\_row\_limit | 名字 | select\_into\_insert\_tablet\_plan\_row\_limit | diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f92c093ddf..35bf672b4f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -683,6 +683,9 @@ public class IoTDBConfig { */ private long continuousQueryMinimumEveryInterval = 1000; + /** How much memory may be used in ONE SELECT INTO operation (in Byte). */ + private long intoOperationBufferSizeInByte = 100 * 1024 * 1024L; + /** * The maximum number of rows can be processed in insert-tablet-plan when executing select-into * statements. @@ -1901,14 +1904,22 @@ public class IoTDBConfig { this.continuousQueryMinimumEveryInterval = minimumEveryInterval; } - public void setSelectIntoInsertTabletPlanRowLimit(int selectIntoInsertTabletPlanRowLimit) { - this.selectIntoInsertTabletPlanRowLimit = selectIntoInsertTabletPlanRowLimit; + public long getIntoOperationBufferSizeInByte() { + return intoOperationBufferSizeInByte; + } + + public void setIntoOperationBufferSizeInByte(long intoOperationBufferSizeInByte) { + this.intoOperationBufferSizeInByte = intoOperationBufferSizeInByte; } public int getSelectIntoInsertTabletPlanRowLimit() { return selectIntoInsertTabletPlanRowLimit; } + public void setSelectIntoInsertTabletPlanRowLimit(int selectIntoInsertTabletPlanRowLimit) { + this.selectIntoInsertTabletPlanRowLimit = selectIntoInsertTabletPlanRowLimit; + } + public int getIntoOperationExecutionThreadCount() { return intoOperationExecutionThreadCount; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c1365a7685..3d94d1b88f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -907,6 +907,11 @@ public class IoTDBDescriptor { // mqtt loadMqttProps(properties); + conf.setIntoOperationBufferSizeInByte( + Long.parseLong( + properties.getProperty( + "into_operation_buffer_size_in_byte", + String.valueOf(conf.getIntoOperationBufferSizeInByte())))); conf.setSelectIntoInsertTabletPlanRowLimit( Integer.parseInt( properties.getProperty( @@ -1467,6 +1472,13 @@ public class IoTDBDescriptor { properties.getProperty( "merge_write_throughput_mb_per_sec", Integer.toString(conf.getCompactionWriteThroughputMbPerSec())))); + + // update select into operation max buffer size + conf.setIntoOperationBufferSizeInByte( + Long.parseLong( + properties.getProperty( + "into_operation_buffer_size_in_byte", + String.valueOf(conf.getIntoOperationBufferSizeInByte())))); // update insert-tablet-plan's row limit for select-into conf.setSelectIntoInsertTabletPlanRowLimit( Integer.parseInt(
