This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 472f169feca [To dev/1.3] Pipe: Changed the default batch memory
request size to avoid CI failure (#15690) (#15704)
472f169feca is described below
commit 472f169fecab89117f77dc0fb680d48e32c111ba
Author: Caideyipi <[email protected]>
AuthorDate: Wed Jun 11 18:23:47 2025 +0800
[To dev/1.3] Pipe: Changed the default batch memory request size to avoid
CI failure (#15690) (#15704)
* Fix
* Update PipeConsensusTransferBatchReqBuilder.java
* Update IoTDBConfig.java
* Update PipeConnectorConstant.java
* Update PipeConnectorConstant.java
* Fix
* Update IoTDBConfig.java
---
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../payload/builder/PipeConsensusTransferBatchReqBuilder.java | 4 ++--
.../src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java | 8 ++++----
.../iotdb/commons/pipe/config/constant/PipeConnectorConstant.java | 3 +--
4 files changed, 8 insertions(+), 9 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index df3c8734967..d429afe529d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -162,7 +162,7 @@ public class IoTDBConfig {
/** Memory allocated for the pipe */
private long allocateMemoryForPipe = Runtime.getRuntime().maxMemory() / 10;
- private long allocateMemoryPerWalCache = 2 * 1024 * 1024;
+ private long allocateMemoryPerWalCache = 512 * 1024;
/** Ratio of memory allocated for buffered arrays */
private double bufferedArraysMemoryProportion = 0.6;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
index 1b022aaeb8f..870cea0ee87 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
@@ -49,8 +49,8 @@ import java.util.Objects;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_DELAY_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_DELAY_MS_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY;
-import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_CONSENSUS_BATCH_SIZE_DEFAULT_VALUE;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_DELAY_DEFAULT_VALUE;
+import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_DELAY_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_DELAY_MS_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_SIZE_KEY;
@@ -94,7 +94,7 @@ public abstract class PipeConsensusTransferBatchReqBuilder
implements AutoClosea
final long requestMaxBatchSizeInBytes =
parameters.getLongOrDefault(
Arrays.asList(CONNECTOR_IOTDB_BATCH_SIZE_KEY,
SINK_IOTDB_BATCH_SIZE_KEY),
- CONNECTOR_IOTDB_CONSENSUS_BATCH_SIZE_DEFAULT_VALUE);
+ CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE);
allocatedMemoryBlock =
PipeDataNodeResourceManager.memory()
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 839e2df2022..e82cc2fb1dd 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -330,10 +330,10 @@ public class CommonConfig {
private float subscriptionCacheMemoryUsagePercentage = 0.2F;
private int subscriptionSubtaskExecutorMaxThreadNum = 2;
- private int subscriptionPrefetchTabletBatchMaxDelayInMs = 1000; // 1s
- private long subscriptionPrefetchTabletBatchMaxSizeInBytes = 16 * MB;
- private int subscriptionPrefetchTsFileBatchMaxDelayInMs = 5000; // 5s
- private long subscriptionPrefetchTsFileBatchMaxSizeInBytes = 80 * MB;
+ private int subscriptionPrefetchTabletBatchMaxDelayInMs = 20; // 1s
+ private long subscriptionPrefetchTabletBatchMaxSizeInBytes = MB;
+ private int subscriptionPrefetchTsFileBatchMaxDelayInMs = 1000; // 5s
+ private long subscriptionPrefetchTsFileBatchMaxSizeInBytes = 2 * MB;
private int subscriptionPollMaxBlockingTimeMs = 500;
private int subscriptionDefaultTimeoutInMs = 10_000; // 10s
private long subscriptionLaunchRetryIntervalMs = 1000;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
index 94cdd935c98..394ff7ae9d6 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
@@ -75,8 +75,7 @@ public class PipeConnectorConstant {
public static final String CONNECTOR_IOTDB_BATCH_SIZE_KEY =
"connector.batch.size-bytes";
public static final String SINK_IOTDB_BATCH_SIZE_KEY =
"sink.batch.size-bytes";
public static final long CONNECTOR_IOTDB_PLAIN_BATCH_SIZE_DEFAULT_VALUE = MB;
- public static final long CONNECTOR_IOTDB_CONSENSUS_BATCH_SIZE_DEFAULT_VALUE
= 16 * MB;
- public static final long CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE =
80 * MB;
+ public static final long CONNECTOR_IOTDB_TS_FILE_BATCH_SIZE_DEFAULT_VALUE =
2 * MB;
public static final String CONNECTOR_IOTDB_USER_KEY = "connector.user";
public static final String SINK_IOTDB_USER_KEY = "sink.user";
