This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a43ef44e224 [chore](routine-load) increase routine load job default
max batch size and rows (#36632)
a43ef44e224 is described below
commit a43ef44e224d2b55491325a3b2687fd6875909f9
Author: hui lai <[email protected]>
AuthorDate: Sat Jun 22 07:55:25 2024 +0800
[chore](routine-load) increase routine load job default max batch size and
rows (#36632)
Most users only care about the size of **max_batch_interval**, but in
order to achieve an interval effect, they have to configure
**max_batch_rows** and **max_batch_size** according to the
characteristics of the data. By adjusting these two default values,
users do not need to worry about configuration in most scenarios.
---
.../main/java/org/apache/doris/load/routineload/RoutineLoadJob.java | 4 ++--
.../java/org/apache/doris/load/routineload/RoutineLoadJobTest.java | 3 +--
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index 2509498f5f4..348ac00f9da 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -109,8 +109,8 @@ public abstract class RoutineLoadJob extends
AbstractTxnStateChangeCallback impl
public static final double DEFAULT_MAX_FILTER_RATIO = 1.0;
public static final long DEFAULT_MAX_INTERVAL_SECOND = 10;
- public static final long DEFAULT_MAX_BATCH_ROWS = 200000;
- public static final long DEFAULT_MAX_BATCH_SIZE = 100 * 1024 * 1024; //
100MB
+ public static final long DEFAULT_MAX_BATCH_ROWS = 20000000;
+ public static final long DEFAULT_MAX_BATCH_SIZE = 1024 * 1024 * 1024; //
1GB
public static final long DEFAULT_EXEC_MEM_LIMIT = 2 * 1024 * 1024 * 1024L;
public static final boolean DEFAULT_STRICT_MODE = false; // default is
false
public static final int DEFAULT_SEND_BATCH_PARALLELISM = 1;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java
index d9494374c03..863cc6807c7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java
@@ -355,7 +355,6 @@ public class RoutineLoadJobTest {
11, "localhost:9092", "test_topic", UserIdentity.ADMIN);
Deencapsulation.setField(routineLoadJob, "maxErrorNum", 10);
Deencapsulation.setField(routineLoadJob, "maxBatchRows", 10);
- Deencapsulation.setField(routineLoadJob, "maxBatchRows", 10);
String showCreateInfo = routineLoadJob.getShowCreateInfo();
String expect = "CREATE ROUTINE LOAD test_load ON 11\n"
+ "WITH APPEND\n"
@@ -366,7 +365,7 @@ public class RoutineLoadJobTest {
+ "\"max_filter_ratio\" = \"1.0\",\n"
+ "\"max_batch_interval\" = \"10\",\n"
+ "\"max_batch_rows\" = \"10\",\n"
- + "\"max_batch_size\" = \"104857600\",\n"
+ + "\"max_batch_size\" = \"1073741824\",\n"
+ "\"format\" = \"csv\",\n"
+ "\"strip_outer_array\" = \"false\",\n"
+ "\"num_as_string\" = \"false\",\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
