This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 28a0217d9d4 [refactor](wg) enable wg by default and init normal wg in
constructor (#31373)
28a0217d9d4 is described below
commit 28a0217d9d438f5890726c8de934574a255d4fab
Author: yiguolei <[email protected]>
AuthorDate: Sun Feb 25 13:47:20 2024 +0800
[refactor](wg) enable wg by default and init normal wg in constructor
(#31373)
should always enable workload group because other operations depend on it
for example MTMV, and spill to disk.
the normal workload group should be created in constructor.
---
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../main/java/org/apache/doris/catalog/Env.java | 2 --
.../resource/workloadgroup/WorkloadGroup.java | 2 +-
.../resource/workloadgroup/WorkloadGroupMgr.java | 36 +++++-----------------
.../workloadgroup/WorkloadGroupMgrTest.java | 32 +++++++++++++------
5 files changed, 31 insertions(+), 43 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 48741df7bb3..541592f2095 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1697,7 +1697,7 @@ public class Config extends ConfigBase {
// enable_workload_group should be immutable and temporarily set to
mutable during the development test phase
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
- public static boolean enable_workload_group = false;
+ public static boolean enable_workload_group = true;
@ConfField(mutable = true)
public static boolean enable_query_queue = true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 4c723b33c0d..630ed5e20ae 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1533,8 +1533,6 @@ public class Env {
editLog.logMasterInfo(masterInfo);
LOG.info("logMasterInfo:{}", masterInfo);
- this.workloadGroupMgr.init();
-
// for master, the 'isReady' is set behind.
// but we are sure that all metadata is replayed if we get here.
// so no need to check 'isReady' flag in this method
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
index e624d29f7f1..0def933c8cf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
@@ -90,7 +90,7 @@ public class WorkloadGroup implements Writable,
GsonPostProcessable {
private int cpuHardLimit = 0;
- private WorkloadGroup(long id, String name, Map<String, String>
properties) {
+ WorkloadGroup(long id, String name, Map<String, String> properties) {
this(id, name, properties, 0);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index 82151d07a8b..60d7e296ed3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -135,6 +135,13 @@ public class WorkloadGroupMgr implements Writable,
GsonPostProcessable {
}
public WorkloadGroupMgr() {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(WorkloadGroup.CPU_SHARE, "1024");
+ properties.put(WorkloadGroup.MEMORY_LIMIT, "30%");
+ properties.put(WorkloadGroup.ENABLE_MEMORY_OVERCOMMIT, "true");
+ WorkloadGroup defaultWorkloadGroup = new WorkloadGroup(1,
DEFAULT_GROUP_NAME, properties);
+ nameToWorkloadGroup.put(DEFAULT_GROUP_NAME, defaultWorkloadGroup);
+ idToWorkloadGroup.put(defaultWorkloadGroup.getId(),
defaultWorkloadGroup);
}
public static WorkloadGroupMgr read(DataInput in) throws IOException {
@@ -158,12 +165,6 @@ public class WorkloadGroupMgr implements Writable,
GsonPostProcessable {
lock.writeLock().unlock();
}
- public void init() {
- if (Config.enable_workload_group || Config.use_fuzzy_session_variable
/* for github workflow */) {
- checkAndCreateDefaultGroup();
- }
- }
-
public List<TPipelineWorkloadGroup> getWorkloadGroup(ConnectContext
context) throws UserException {
String groupName = getWorkloadGroupNameAndCheckPriv(context);
List<TPipelineWorkloadGroup> workloadGroups = Lists.newArrayList();
@@ -239,29 +240,6 @@ public class WorkloadGroupMgr implements Writable,
GsonPostProcessable {
return groupName;
}
- private void checkAndCreateDefaultGroup() {
- WorkloadGroup defaultWorkloadGroup = null;
- writeLock();
- try {
- if (nameToWorkloadGroup.containsKey(DEFAULT_GROUP_NAME)) {
- return;
- }
- Map<String, String> properties = Maps.newHashMap();
- properties.put(WorkloadGroup.CPU_SHARE, "1024");
- properties.put(WorkloadGroup.MEMORY_LIMIT, "30%");
- properties.put(WorkloadGroup.ENABLE_MEMORY_OVERCOMMIT, "true");
- defaultWorkloadGroup = WorkloadGroup.create(DEFAULT_GROUP_NAME,
properties);
- nameToWorkloadGroup.put(DEFAULT_GROUP_NAME, defaultWorkloadGroup);
- idToWorkloadGroup.put(defaultWorkloadGroup.getId(),
defaultWorkloadGroup);
-
Env.getCurrentEnv().getEditLog().logCreateWorkloadGroup(defaultWorkloadGroup);
- } catch (DdlException e) {
- LOG.warn("Create workload group " + DEFAULT_GROUP_NAME + " fail");
- } finally {
- writeUnlock();
- }
- LOG.info("Create workload group success: {}", defaultWorkloadGroup);
- }
-
public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws
DdlException {
WorkloadGroup workloadGroup =
WorkloadGroup.create(stmt.getWorkloadGroupName(), stmt.getProperties());
String workloadGroupName = workloadGroup.getName();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java
index e4501fd6463..1a9fac1ea7a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java
@@ -120,13 +120,13 @@ public class WorkloadGroupMgrTest {
workloadGroupMgr.createWorkloadGroup(stmt1);
Map<String, WorkloadGroup> nameToRG =
workloadGroupMgr.getNameToWorkloadGroup();
- Assert.assertEquals(1, nameToRG.size());
+ Assert.assertEquals(2, nameToRG.size());
Assert.assertTrue(nameToRG.containsKey(name1));
WorkloadGroup group1 = nameToRG.get(name1);
Assert.assertEquals(name1, group1.getName());
Map<Long, WorkloadGroup> idToRG =
workloadGroupMgr.getIdToWorkloadGroup();
- Assert.assertEquals(1, idToRG.size());
+ Assert.assertEquals(2, idToRG.size());
Assert.assertTrue(idToRG.containsKey(group1.getId()));
Map<String, String> properties2 = Maps.newHashMap();
@@ -137,11 +137,11 @@ public class WorkloadGroupMgrTest {
workloadGroupMgr.createWorkloadGroup(stmt2);
nameToRG = workloadGroupMgr.getNameToWorkloadGroup();
- Assert.assertEquals(2, nameToRG.size());
+ Assert.assertEquals(3, nameToRG.size());
Assert.assertTrue(nameToRG.containsKey(name2));
WorkloadGroup group2 = nameToRG.get(name2);
idToRG = workloadGroupMgr.getIdToWorkloadGroup();
- Assert.assertEquals(2, idToRG.size());
+ Assert.assertEquals(3, idToRG.size());
Assert.assertTrue(idToRG.containsKey(group2.getId()));
try {
@@ -153,8 +153,8 @@ public class WorkloadGroupMgrTest {
CreateWorkloadGroupStmt stmt3 = new CreateWorkloadGroupStmt(true,
name2, properties2);
workloadGroupMgr.createWorkloadGroup(stmt3);
- Assert.assertEquals(2, workloadGroupMgr.getIdToWorkloadGroup().size());
- Assert.assertEquals(2,
workloadGroupMgr.getNameToWorkloadGroup().size());
+ Assert.assertEquals(3, workloadGroupMgr.getIdToWorkloadGroup().size());
+ Assert.assertEquals(3,
workloadGroupMgr.getNameToWorkloadGroup().size());
}
@Test
@@ -170,8 +170,14 @@ public class WorkloadGroupMgrTest {
workloadGroupMgr.createWorkloadGroup(stmt1);
context.getSessionVariable().setWorkloadGroup(name1);
List<TopicInfo> tWorkloadGroups1 =
workloadGroupMgr.getPublishTopicInfo();
- Assert.assertEquals(1, tWorkloadGroups1.size());
- TopicInfo tWorkloadGroup1 = tWorkloadGroups1.get(0);
+ Assert.assertEquals(2, tWorkloadGroups1.size());
+ TopicInfo tWorkloadGroup1 = null;
+ for (int i = 0; i < 2; ++i) {
+ if
(tWorkloadGroups1.get(i).getWorkloadGroupInfo().getName().equals(name1)) {
+ tWorkloadGroup1 = tWorkloadGroups1.get(i);
+ break;
+ }
+ }
Assert.assertEquals(name1,
tWorkloadGroup1.getWorkloadGroupInfo().getName());
Assert.assertTrue(tWorkloadGroup1.getWorkloadGroupInfo().getCpuShare()
== 10);
@@ -243,8 +249,14 @@ public class WorkloadGroupMgrTest {
context.getSessionVariable().setWorkloadGroup(name);
List<TopicInfo> tWorkloadGroups =
workloadGroupMgr.getPublishTopicInfo();
- Assert.assertEquals(1, tWorkloadGroups.size());
- TopicInfo tWorkloadGroup1 = tWorkloadGroups.get(0);
+ Assert.assertEquals(2, tWorkloadGroups.size());
+ TopicInfo tWorkloadGroup1 = null;
+ for (int i = 0; i < 2; ++i) {
+ if
(tWorkloadGroups.get(i).getWorkloadGroupInfo().getName().equals(name)) {
+ tWorkloadGroup1 = tWorkloadGroups.get(i);
+ break;
+ }
+ }
Assert.assertTrue(tWorkloadGroup1.getWorkloadGroupInfo().getCpuShare()
== 5);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
