This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 27f6e4649e [improvement](stats) Catch exception properly #22503
27f6e4649e is described below
commit 27f6e4649ed101a6003dca0f4efdff6b01d42d8f
Author: AKIRA <[email protected]>
AuthorDate: Thu Aug 3 15:16:55 2023 +0800
[improvement](stats) Catch exception properly #22503
Catch exception instead of throw to caller directly to avoid unexpected
interruption of upper logic
---
.../apache/doris/statistics/AnalysisManager.java | 13 +++--
.../doris/statistics/StatisticsAutoAnalyzer.java | 59 +++++++++++++---------
2 files changed, 44 insertions(+), 28 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 9a600a5112..24e64ceabb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -174,8 +174,7 @@ public class AnalysisManager extends Daemon implements
Writable {
}
}
- public List<AnalysisInfo> buildAnalysisInfosForDB(DatabaseIf<TableIf> db,
AnalyzeProperties analyzeProperties)
- throws DdlException {
+ public List<AnalysisInfo> buildAnalysisInfosForDB(DatabaseIf<TableIf> db,
AnalyzeProperties analyzeProperties) {
List<TableIf> tbls = db.getTables();
List<AnalysisInfo> analysisInfos = new ArrayList<>();
db.readLock();
@@ -195,12 +194,18 @@ public class AnalysisManager extends Daemon implements
Writable {
try {
analyzeTblStmt.check();
} catch (AnalysisException analysisException) {
- throw new DdlException(analysisException.getMessage(),
analysisException);
+ LOG.warn("Failed to build analyze job: {}",
+ analysisException.getMessage(), analysisException);
}
analyzeStmts.add(analyzeTblStmt);
}
for (AnalyzeTblStmt analyzeTblStmt : analyzeStmts) {
- analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
+ try {
+ analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
+ } catch (DdlException e) {
+ LOG.warn("Failed to build analyze job: {}",
+ e.getMessage(), e);
+ }
}
} finally {
db.readUnlock();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
index 02fe96ec71..ebe687d78e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
@@ -73,8 +73,8 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
return;
}
+ analyzePeriodically();
if (!Config.enable_full_auto_analyze) {
- analyzePeriodically();
analyzeAutomatically();
} else {
analyzeAll();
@@ -85,26 +85,28 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
private void analyzeAll() {
Set<CatalogIf> catalogs =
Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog();
for (CatalogIf ctl : catalogs) {
- try {
- Collection<DatabaseIf> dbs = ctl.getAllDbs();
- for (DatabaseIf<TableIf> databaseIf : dbs) {
- if
(StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName()))
{
+
+ Collection<DatabaseIf> dbs = ctl.getAllDbs();
+ for (DatabaseIf<TableIf> databaseIf : dbs) {
+ if
(StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName()))
{
+ continue;
+ }
+ AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
+ List<AnalysisInfo> analysisInfos =
analysisManager.buildAnalysisInfosForDB(databaseIf,
+ AnalyzeProperties.DEFAULT_PROP);
+ for (AnalysisInfo analysisInfo : analysisInfos) {
+ analysisInfo = getReAnalyzeRequiredPart(analysisInfo);
+ if (analysisInfo == null) {
continue;
}
- AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
- List<AnalysisInfo> analysisInfos =
analysisManager.buildAnalysisInfosForDB(databaseIf,
- AnalyzeProperties.DEFAULT_PROP);
- for (AnalysisInfo analysisInfo : analysisInfos) {
- analysisInfo = getReAnalyzeRequiredPart(analysisInfo);
- if (analysisInfo == null) {
- continue;
- }
+ try {
analysisManager.createSystemAnalysisJob(analysisInfo,
analysisTaskExecutor);
+ } catch (Exception e) {
+ LOG.warn("Failed to create analysis job", e);
}
}
- } catch (Throwable t) {
- LOG.warn("Failed to analyze all statistics.", t);
}
+
}
analyzePeriodically();
@@ -160,16 +162,20 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
* @return new job info after check
* @throws Throwable failed to check
*/
- private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) throws
Throwable {
+ private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
long lastExecTimeInMs = jobInfo.lastExecTimeInMs;
TableIf table = StatisticsUtil
.findTable(jobInfo.catalogName, jobInfo.dbName,
jobInfo.tblName);
- TableStatistic tblStats =
StatisticsRepository.fetchTableLevelStats(table.getId());
+ TableStatistic tblStats = null;
+ try {
+ tblStats =
StatisticsRepository.fetchTableLevelStats(table.getId());
+ } catch (Throwable t) {
+ LOG.warn("Failed to fetch table stats", t);
+ return null;
+ }
if (tblStats == TableStatistic.UNKNOWN) {
- LOG.warn("Failed to automatically analyze statistics, "
- + "no corresponding table statistics for job: {}",
jobInfo.toString());
- throw new DdlException("No corresponding table statistics for
automatic job.");
+ return jobInfo;
}
if (!needReanalyzeTable(table, tblStats)) {
@@ -200,7 +206,7 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
}
private void checkAnalyzedPartitions(TableIf table, Set<String>
statsPartitions,
- Set<String> needRunPartitions, long lastExecTimeInMs) throws
DdlException {
+ Set<String> needRunPartitions, long lastExecTimeInMs) {
for (String statsPartition : statsPartitions) {
Partition partition = table.getPartition(statsPartition);
if (partition == null) {
@@ -209,12 +215,17 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
needRunPartitions.add(statsPartition);
continue;
}
- TableStatistic partitionStats = StatisticsRepository
+ TableStatistic partitionStats = null;
+ try {
+ partitionStats = StatisticsRepository
.fetchTableLevelOfPartStats(partition.getId());
- if (partitionStats == TableStatistic.UNKNOWN) {
+ } catch (DdlException e) {
+ LOG.warn("Failed to fetch part stats", e);
continue;
}
- if (needReanalyzePartition(lastExecTimeInMs, partition,
partitionStats)) {
+
+ if (needReanalyzePartition(lastExecTimeInMs, partition,
partitionStats)
+ || partitionStats == TableStatistic.UNKNOWN) {
needRunPartitions.add(partition.getName());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
