This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 19c36dcc866 [Fix](statistics)Fix auto job start time incorrect bug
(#27402)
19c36dcc866 is described below
commit 19c36dcc86615bb2b54d268054d10a85ba25ae9a
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Nov 22 21:38:08 2023 +0800
[Fix](statistics)Fix auto job start time incorrect bug (#27402)
Before, the auto analyze job start time was the job creation time, not the
start to execute time, which is inaccurate. This pr is to change the start time
to the first task start to execute time.
---
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
.../org/apache/doris/statistics/AnalysisInfo.java | 7 +++
.../apache/doris/statistics/AnalysisManager.java | 9 +++-
.../doris/statistics/AnalysisManagerTest.java | 51 ++++++++++++++++++++++
4 files changed, 67 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 8f92115695e..65440ac00dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2662,7 +2662,7 @@ public class ShowExecutor {
}
row.add(analysisInfo.scheduleType.toString());
LocalDateTime startTime =
-
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.createTime),
+
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.startTime),
java.time.ZoneId.systemDefault());
LocalDateTime endTime =
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.endTime),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index ec0980d16b0..0606350b8be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -185,6 +185,9 @@ public class AnalysisInfo implements Writable {
@SerializedName("createTime")
public final long createTime = System.currentTimeMillis();
+ @SerializedName("startTime")
+ public long startTime;
+
@SerializedName("endTime")
public long endTime;
/**
@@ -330,6 +333,10 @@ public class AnalysisInfo implements Writable {
return analysisInfo;
}
+ public void markStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
public void markFinished() {
state = AnalysisState.FINISHED;
endTime = System.currentTimeMillis();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index dbb5dd668a9..c6deaf0268a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -158,6 +158,7 @@ public class AnalysisManager implements Writable {
// Set the job state to RUNNING when its first task becomes
RUNNING.
if (info.state.equals(AnalysisState.RUNNING) &&
job.state.equals(AnalysisState.PENDING)) {
job.state = AnalysisState.RUNNING;
+ job.markStartTime(System.currentTimeMillis());
replayCreateAnalysisJob(job);
}
boolean allFinished = true;
@@ -200,6 +201,13 @@ public class AnalysisManager implements Writable {
if (job == null) {
return null;
}
+ synchronized (job) {
+ // Set the job state to RUNNING when its first task becomes
RUNNING.
+ if (info.state.equals(AnalysisState.RUNNING) &&
job.state.equals(AnalysisState.PENDING)) {
+ job.state = AnalysisState.RUNNING;
+ job.markStartTime(System.currentTimeMillis());
+ }
+ }
int failedCount = 0;
StringJoiner reason = new StringJoiner(", ");
Map<Long, BaseAnalysisTask> taskMap =
analysisJobIdToTaskMap.get(info.jobId);
@@ -1002,7 +1010,6 @@ public class AnalysisManager implements Writable {
}
public void registerSysJob(AnalysisInfo jobInfo, Map<Long,
BaseAnalysisTask> taskInfos) {
- jobInfo.state = AnalysisState.RUNNING;
systemJobInfoMap.put(jobInfo.jobId, jobInfo);
analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 345d8ee2cfd..35f02b88115 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -322,6 +322,57 @@ public class AnalysisManagerTest {
Assertions.assertTrue(analysisManager.systemJobInfoMap.isEmpty());
}
+ @Test
+ public void testSystemJobStartTime() {
+ new MockUp<BaseAnalysisTask>() {
+
+ @Mock
+ protected void init(AnalysisInfo info) {
+
+ }
+ };
+
+ new MockUp<AnalysisManager>() {
+ @Mock
+ public void updateTableStats(AnalysisInfo jobInfo) {
+ }
+
+ @Mock
+ protected void logAutoJob(AnalysisInfo autoJob) {
+
+ }
+ };
+
+ AnalysisManager analysisManager = new AnalysisManager();
+ AnalysisInfo job = new AnalysisInfoBuilder()
+ .setJobId(0)
+ .setColName("col1, col2").build();
+ analysisManager.systemJobInfoMap.put(job.jobId, job);
+ AnalysisInfo task1 = new AnalysisInfoBuilder()
+ .setJobId(0)
+ .setTaskId(1)
+ .setState(AnalysisState.PENDING)
+ .setColName("col1").build();
+ AnalysisInfo task2 = new AnalysisInfoBuilder()
+ .setJobId(0)
+ .setTaskId(1)
+ .setState(AnalysisState.PENDING)
+ .setColName("col2").build();
+ OlapAnalysisTask ot1 = new OlapAnalysisTask(task1);
+ OlapAnalysisTask ot2 = new OlapAnalysisTask(task2);
+ Map<Long, BaseAnalysisTask> taskMap = new HashMap<>();
+ taskMap.put(ot1.info.taskId, ot1);
+ taskMap.put(ot2.info.taskId, ot2);
+ analysisManager.analysisJobIdToTaskMap.put(job.jobId, taskMap);
+
+ job.state = AnalysisState.PENDING;
+ long l = System.currentTimeMillis();
+ analysisManager.systemJobInfoMap.put(job.jobId, job);
+ analysisManager.systemJobStatusUpdater.apply(new
TaskStatusWrapper(task1,
+ AnalysisState.RUNNING, "", 0));
+ Assertions.assertTrue(job.startTime >= l);
+ }
+
@Test
public void testReAnalyze() {
new MockUp<OlapTable>() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
