This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new e5f57eff603 branch-3.1: [opt](wg) remove wg sum limit check (#52227)
e5f57eff603 is described below
commit e5f57eff603628f0045f9a94138686d1e3cb1fa3
Author: wangbo <[email protected]>
AuthorDate: Tue Jun 24 23:41:24 2025 +0800
branch-3.1: [opt](wg) remove wg sum limit check (#52227)
---
be/src/agent/workload_group_listener.cpp | 7 +----
be/src/runtime/workload_group/workload_group.cpp | 33 ++++++----------------
.../workload_group/workload_group_manager.h | 6 ----
.../main/java/org/apache/doris/common/Config.java | 7 +++--
.../resource/workloadgroup/WorkloadGroup.java | 9 ------
.../resource/workloadgroup/WorkloadGroupMgr.java | 13 +--------
6 files changed, 14 insertions(+), 61 deletions(-)
diff --git a/be/src/agent/workload_group_listener.cpp
b/be/src/agent/workload_group_listener.cpp
index f0f57869f25..50a8b8e509c 100644
--- a/be/src/agent/workload_group_listener.cpp
+++ b/be/src/agent/workload_group_listener.cpp
@@ -54,10 +54,6 @@ void WorkloadGroupListener::handle_topic_info(const
std::vector<TopicInfo>& topi
auto wg =
_exec_env->workload_group_mgr()->get_or_create_workload_group(workload_group_info);
- // 3 set cpu soft hard limit switch
- _exec_env->workload_group_mgr()->_enable_cpu_hard_limit.store(
- workload_group_info.enable_cpu_hard_limit);
-
// 4 create and update task scheduler
wg->upsert_task_scheduler(&workload_group_info, _exec_env);
@@ -65,8 +61,7 @@ void WorkloadGroupListener::handle_topic_info(const
std::vector<TopicInfo>& topi
wg->upsert_scan_io_throttle(&workload_group_info);
LOG(INFO) << "[topic_publish_wg]update workload group finish, wg info="
- << wg->debug_string() << ", enable_cpu_hard_limit="
- << (_exec_env->workload_group_mgr()->enable_cpu_hard_limit()
? "true" : "false")
+ << wg->debug_string()
<< ", cgroup cpu_shares=" <<
workload_group_info.cgroup_cpu_shares
<< ", cgroup cpu_hard_limit=" <<
workload_group_info.cgroup_cpu_hard_limit
<< ", cgroup home path=" << config::doris_cgroup_cpu_path
diff --git a/be/src/runtime/workload_group/workload_group.cpp
b/be/src/runtime/workload_group/workload_group.cpp
index b07548f9970..ba84e5786ce 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -47,7 +47,6 @@ namespace doris {
const static std::string MEMORY_LIMIT_DEFAULT_VALUE = "0%";
const static bool ENABLE_MEMORY_OVERCOMMIT_DEFAULT_VALUE = true;
-const static int CPU_HARD_LIMIT_DEFAULT_VALUE = -1;
const static int SPILL_LOW_WATERMARK_DEFAULT_VALUE = 50;
const static int SPILL_HIGH_WATERMARK_DEFAULT_VALUE = 80;
@@ -343,7 +342,12 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info(
}
// 5 cpu hard limit
- int cpu_hard_limit = CPU_HARD_LIMIT_DEFAULT_VALUE;
+ // cgroup v1 and v2 has different default cpu quota value,
+ // v1's default value is -1,
+ // v2's default value is 'max 100000',
+ // it's hard to unify them, so set -1 here means it's an invalid value,
+ // it could be replaced to default value when write value to cpu file,
refer modify_cg_cpu_hard_limit_no_lock
+ int cpu_hard_limit = -1;
if (tworkload_group_info.__isset.cpu_hard_limit &&
tworkload_group_info.cpu_hard_limit > 0) {
cpu_hard_limit = tworkload_group_info.cpu_hard_limit;
}
@@ -363,12 +367,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info(
enable_memory_overcommit =
tworkload_group_info.enable_memory_overcommit;
}
- // 8 cpu soft limit or hard limit
- bool enable_cpu_hard_limit = false;
- if (tworkload_group_info.__isset.enable_cpu_hard_limit) {
- enable_cpu_hard_limit = tworkload_group_info.enable_cpu_hard_limit;
- }
-
// 9 scan thread num
int scan_thread_num = config::doris_scanner_thread_pool_thread_num;
if (tworkload_group_info.__isset.scan_thread_num &&
tworkload_group_info.scan_thread_num > 0) {
@@ -422,7 +420,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info(
.enable_memory_overcommit = enable_memory_overcommit,
.version = version,
.cpu_hard_limit = cpu_hard_limit,
- .enable_cpu_hard_limit = enable_cpu_hard_limit,
.scan_thread_num = scan_thread_num,
.max_remote_scan_thread_num = max_remote_scan_thread_num,
.min_remote_scan_thread_num = min_remote_scan_thread_num,
@@ -437,7 +434,6 @@ void
WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e
std::string tg_name = tg_info->name;
int cpu_hard_limit = tg_info->cpu_hard_limit;
uint64_t cpu_shares = tg_info->cpu_share;
- bool enable_cpu_hard_limit = tg_info->enable_cpu_hard_limit;
int scan_thread_num = tg_info->scan_thread_num;
int max_remote_scan_thread_num = tg_info->max_remote_scan_thread_num;
int min_remote_scan_thread_num = tg_info->min_remote_scan_thread_num;
@@ -551,21 +547,8 @@ void
WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e
// step 6: update cgroup cpu if needed
if (_cgroup_cpu_ctl) {
- if (enable_cpu_hard_limit) {
- if (cpu_hard_limit > 0) {
- _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit);
- _cgroup_cpu_ctl->update_cpu_soft_limit(
- CgroupCpuCtl::cpu_soft_limit_default_value());
- } else {
- LOG(INFO) << "[upsert wg thread pool] enable cpu hard limit
but value is "
- "illegal: "
- << cpu_hard_limit << ", gid=" << tg_id;
- }
- } else {
- _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares);
- _cgroup_cpu_ctl->update_cpu_hard_limit(
- CPU_HARD_LIMIT_DEFAULT_VALUE); // disable cpu hard limit
- }
+ _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit);
+ _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares);
_cgroup_cpu_ctl->get_cgroup_cpu_info(&(tg_info->cgroup_cpu_shares),
&(tg_info->cgroup_cpu_hard_limit));
}
diff --git a/be/src/runtime/workload_group/workload_group_manager.h
b/be/src/runtime/workload_group/workload_group_manager.h
index db2444e0bc1..3cd00ebdb44 100644
--- a/be/src/runtime/workload_group/workload_group_manager.h
+++ b/be/src/runtime/workload_group/workload_group_manager.h
@@ -54,12 +54,6 @@ public:
void stop();
- std::atomic<bool> _enable_cpu_hard_limit = false;
-
- bool enable_cpu_soft_limit() { return !_enable_cpu_hard_limit.load(); }
-
- bool enable_cpu_hard_limit() { return _enable_cpu_hard_limit.load(); }
-
void refresh_wg_weighted_memory_limit();
void get_wg_resource_usage(vectorized::Block* block);
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 8a66c885291..2700293635a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1919,6 +1919,10 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_workload_group = true;
+
+ @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
+ public static boolean enable_wg_memory_sum_limit = true;
+
@ConfField(mutable = true)
public static boolean enable_query_queue = true;
@@ -1929,9 +1933,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static long query_queue_update_interval_ms = 5000;
- @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
- public static boolean enable_cpu_hard_limit = false;
-
@ConfField(mutable = true, description = {
"当BE内存用量大于该值时，查询会进入排队逻辑，默认值为-1，代表该值不生效。取值范围0~1的小数",
"When be memory usage bigger than this value, query could queue, "
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
index 408895dd12f..4fff8388dec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
@@ -19,7 +19,6 @@ package org.apache.doris.resource.workloadgroup;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.Pair;
@@ -593,14 +592,6 @@ public class WorkloadGroup implements Writable,
GsonPostProcessable {
if (memOvercommitStr != null) {
tWorkloadGroupInfo.setEnableMemoryOvercommit(Boolean.valueOf(memOvercommitStr));
}
- // enable_cpu_hard_limit = true, using cpu hard limit
- // enable_cpu_hard_limit = false, using cpu soft limit
- tWorkloadGroupInfo.setEnableCpuHardLimit(Config.enable_cpu_hard_limit);
-
- if (Config.enable_cpu_hard_limit && cpuHardLimit <= 0) {
- LOG.warn("enable_cpu_hard_limit=true but cpuHardLimit value not
illegal,"
- + "id=" + id + ",name=" + name);
- }
String scanThreadNumStr = properties.get(SCAN_THREAD_NUM);
if (scanThreadNumStr != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index 442b1bd783c..1e81c91bcea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -390,7 +390,6 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
for (String newWgOneTag : newWgTagSet) {
double sumOfAllMemLimit = 0;
- int sumOfAllCpuHardLimit = 0;
// 1 get sum value of all wg which has same tag without current wg
for (Map.Entry<Long, WorkloadGroup> entry :
idToWorkloadGroup.entrySet()) {
@@ -409,9 +408,6 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
continue;
}
- if (wg.getCpuHardLimitWhenCalSum() > 0) {
- sumOfAllCpuHardLimit += wg.getCpuHardLimitWhenCalSum();
- }
if (wg.getMemoryLimitPercentWhenCalSum() > 0) {
sumOfAllMemLimit += wg.getMemoryLimitPercentWhenCalSum();
}
@@ -419,22 +415,15 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
// 2 sum current wg value
sumOfAllMemLimit += newWg.getMemoryLimitPercentWhenCalSum();
- sumOfAllCpuHardLimit += newWg.getCpuHardLimitWhenCalSum();
// 3 check total sum
- if (sumOfAllMemLimit > 100.0 + 1e-6) {
+ if (Config.enable_wg_memory_sum_limit && sumOfAllMemLimit > 100.0
+ 1e-6) {
throw new DdlException(
"The sum of all workload group " +
WorkloadGroup.MEMORY_LIMIT + " within tag " + (
newWgTag.isPresent() ? newWgTag.get() : "")
+ " cannot be greater than 100.0%. current sum
val:" + sumOfAllMemLimit);
}
- if (sumOfAllCpuHardLimit > 100) {
- throw new DdlException(
- "sum of all workload group " +
WorkloadGroup.CPU_HARD_LIMIT + " within tag " + (
- newWgTag.isPresent()
- ? newWgTag.get() : "") + " can not be
greater than 100% ");
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
