This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new e1971e7259 [enhancement](load) make default load mem limit
configurable (#12348)
e1971e7259 is described below
commit e1971e725962663ea5b8db099a7938cc958943d6
Author: Henry2SS <[email protected]>
AuthorDate: Mon Sep 12 10:25:01 2022 +0800
[enhancement](load) make default load mem limit configurable (#12348)
* make LoadMemLimit valid for broker load, stream load and routine load
Co-authored-by: wuhangze <[email protected]>
---
be/src/runtime/runtime_state.cpp | 1 +
fe/fe-core/src/main/java/org/apache/doris/load/Load.java | 2 +-
.../main/java/org/apache/doris/load/routineload/RoutineLoadJob.java | 2 ++
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java | 3 +++
5 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 0446a303b0..4036a99786 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -445,6 +445,7 @@ Status RuntimeState::StartSpilling(MemTracker* mem_tracker)
{
}
int64_t RuntimeState::get_load_mem_limit() {
+ // TODO: the code is abandoned, it can be deleted after v1.3
if (_query_options.__isset.load_mem_limit && _query_options.load_mem_limit
> 0) {
return _query_options.load_mem_limit;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index f2fe394e72..c6afd572b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -439,7 +439,7 @@ public class Load {
// resource info
if (ConnectContext.get() != null) {
job.setResourceInfo(ConnectContext.get().toResourceCtx());
-
job.setExecMemLimit(ConnectContext.get().getSessionVariable().getMaxExecMemByte());
+
job.setExecMemLimit(ConnectContext.get().getSessionVariable().getLoadMemLimit());
}
// job properties
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index ab87e4b180..296d40e550 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -290,6 +290,8 @@ public abstract class RoutineLoadJob extends
AbstractTxnStateChangeCallback impl
}
if (stmt.getExecMemLimit() != -1) {
this.execMemLimit = stmt.getExecMemLimit();
+ } else if (ConnectContext.get() != null) {
+ this.execMemLimit =
ConnectContext.get().getSessionVariable().getLoadMemLimit();
}
if (stmt.getSendBatchParallelism() > 0) {
this.sendBatchParallelism = stmt.getSendBatchParallelism();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d051c632f2..6af37a639b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -358,7 +358,7 @@ public class SessionVariable implements Serializable,
Writable {
public boolean forwardToMaster = true;
@VariableMgr.VarAttr(name = LOAD_MEM_LIMIT)
- public long loadMemLimit = 0L;
+ public long loadMemLimit = 2 * 1024 * 1024 * 1024L; // 2GB as default
@VariableMgr.VarAttr(name = USE_V2_ROLLUP)
public boolean useV2Rollup = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
index 248b7f127d..b475757653 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
@@ -30,6 +30,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.load.loadv2.LoadTask;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileType;
import org.apache.doris.thrift.TStreamLoadPutRequest;
@@ -272,6 +273,8 @@ public class StreamLoadTask implements LoadTaskInfo {
}
if (request.isSetExecMemLimit()) {
execMemLimit = request.getExecMemLimit();
+ } else {
+ execMemLimit =
VariableMgr.getDefaultSessionVariable().getLoadMemLimit();
}
if (request.getFormatType() == TFileFormatType.FORMAT_JSON) {
if (request.getJsonpaths() != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
