This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94b75515e55 [minor](stats) Throw error when sync analyze failed
(#27845)
94b75515e55 is described below
commit 94b75515e558aca418c0a5e9f8a6d4d7c03c8fa8
Author: AKIRA <[email protected]>
AuthorDate: Fri Dec 1 15:44:27 2023 +0800
[minor](stats) Throw error when sync analyze failed (#27845)
---
.../org/apache/doris/statistics/AnalysisJob.java | 25 ++++++----------------
.../apache/doris/statistics/BaseAnalysisTask.java | 4 ++--
.../apache/doris/statistics/StatisticsCleaner.java | 17 +++++++++++----
.../apache/doris/statistics/AnalysisJobTest.java | 4 ++--
4 files changed, 24 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
index 41e4b6b317e..bb51ae593ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
@@ -119,7 +119,7 @@ public class AnalysisJob {
if (killed) {
return;
}
- // buf could be empty when nothing need to do, for example user submit
an analysis task for table with no data
+ // buf could be empty when nothing need to do,r for example user
submit an analysis task for table with no data
// change
if (!buf.isEmpty()) {
String insertStmt = "INSERT INTO " +
StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME + " VALUES ";
@@ -128,28 +128,17 @@ public class AnalysisJob {
values.add(data.toSQL(true));
}
insertStmt += values.toString();
- int retryTimes = 0;
- while (retryTimes < StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
- if (killed) {
- return;
- }
- try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext(false)) {
- stmtExecutor = new StmtExecutor(r.connectContext,
insertStmt);
- executeWithExceptionOnFail(stmtExecutor);
- break;
- } catch (Exception t) {
- LOG.warn("Failed to write buf: " + insertStmt, t);
- retryTimes++;
- if (retryTimes >=
StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
- updateTaskState(AnalysisState.FAILED, t.getMessage());
- return;
- }
- }
+ try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext(false)) {
+ stmtExecutor = new StmtExecutor(r.connectContext, insertStmt);
+ executeWithExceptionOnFail(stmtExecutor);
+ } catch (Exception t) {
+ throw new RuntimeException("Failed to analyze: " +
t.getMessage());
}
}
updateTaskState(AnalysisState.FINISHED, "");
syncLoadStats();
queryFinished.clear();
+ buf.clear();
}
protected void executeWithExceptionOnFail(StmtExecutor stmtExecutor)
throws Exception {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 4c0e07ce6cb..34d362161fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -181,7 +181,7 @@ public abstract class BaseAnalysisTask {
protected void executeWithRetry() {
int retriedTimes = 0;
- while (retriedTimes <= StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
+ while (retriedTimes < StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
if (killed) {
break;
}
@@ -193,7 +193,7 @@ public abstract class BaseAnalysisTask {
throw new RuntimeException(t);
}
LOG.warn("Failed to execute analysis task, retried times: {}",
retriedTimes++, t);
- if (retriedTimes >
StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
+ if (retriedTimes >=
StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
job.taskFailed(this, t.getMessage());
throw new RuntimeException(t);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index 88fa098e57b..21deb44c0a4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -75,11 +75,20 @@ public class StatisticsCleaner extends MasterDaemon {
}
public synchronized void clear() {
- if (!init()) {
- return;
+ try {
+ if (!init()) {
+ return;
+ }
+ clearStats(colStatsTbl);
+ clearStats(histStatsTbl);
+ } finally {
+ colStatsTbl = null;
+ histStatsTbl = null;
+ idToCatalog = null;
+ idToDb = null;
+ idToTbl = null;
+ idToMVIdx = null;
}
- clearStats(colStatsTbl);
- clearStats(histStatsTbl);
}
private void clearStats(OlapTable statsTbl) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
index bca05d8299c..255ab7106aa 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
@@ -206,7 +206,7 @@ public class AnalysisJobTest {
@Mock
protected void executeWithExceptionOnFail(StmtExecutor
stmtExecutor) throws Exception {
- throw new RuntimeException();
+ // DO NOTHING
}
@Mock
@@ -218,7 +218,7 @@ public class AnalysisJobTest {
job.queryFinished = new HashSet<>();
job.queryFinished.add(task2);
job.writeBuf();
- Assertions.assertEquals(1, job.queryFinished.size());
+ Assertions.assertEquals(0, job.queryFinished.size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
