This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a89edfe784 [improvement](stats) Catch exception properly (#22503)
(#22896)
a89edfe784 is described below
commit a89edfe784b511df6077c7f63f16052c97080b57
Author: AKIRA <[email protected]>
AuthorDate: Fri Aug 11 19:31:56 2023 +0800
[improvement](stats) Catch exception properly (#22503) (#22896)
---
.../apache/doris/statistics/AnalysisManager.java | 27 +++++---
.../doris/statistics/StatisticsAutoAnalyzer.java | 74 +++++++++++++++++++---
2 files changed, 85 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index e549bd6b0b..ef909728c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AnalyzeDBStmt;
+import org.apache.doris.analysis.AnalyzeProperties;
import org.apache.doris.analysis.AnalyzeStmt;
import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.DropAnalyzeJobStmt;
@@ -169,6 +170,13 @@ public class AnalysisManager extends Daemon implements
Writable {
public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy)
throws DdlException {
DatabaseIf<TableIf> db = analyzeDBStmt.getDb();
+ List<AnalysisInfo> analysisInfos = buildAnalysisInfosForDB(db,
analyzeDBStmt.getAnalyzeProperties());
+ if (!analyzeDBStmt.isSync()) {
+ sendJobId(analysisInfos, proxy);
+ }
+ }
+
+ public List<AnalysisInfo> buildAnalysisInfosForDB(DatabaseIf<TableIf> db,
AnalyzeProperties analyzeProperties) {
List<TableIf> tbls = db.getTables();
List<AnalysisInfo> analysisInfos = new ArrayList<>();
db.readLock();
@@ -178,27 +186,30 @@ public class AnalysisManager extends Daemon implements
Writable {
if (table instanceof View) {
continue;
}
- TableName tableName = new
TableName(analyzeDBStmt.getCtlIf().getName(), db.getFullName(),
+ TableName tableName = new TableName(db.getCatalog().getName(),
db.getFullName(),
table.getName());
- AnalyzeTblStmt analyzeTblStmt = new
AnalyzeTblStmt(analyzeDBStmt.getAnalyzeProperties(), tableName,
+ AnalyzeTblStmt analyzeTblStmt = new
AnalyzeTblStmt(analyzeProperties, tableName,
null, db.getId(), table);
try {
analyzeTblStmt.check();
} catch (AnalysisException analysisException) {
- throw new DdlException(analysisException.getMessage(),
analysisException);
+ LOG.warn("Failed to build analyze job: {}",
+ analysisException.getMessage(), analysisException);
}
analyzeStmts.add(analyzeTblStmt);
}
for (AnalyzeTblStmt analyzeTblStmt : analyzeStmts) {
- analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
- }
- if (!analyzeDBStmt.isSync()) {
- sendJobId(analysisInfos, proxy);
+ try {
+ analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
+ } catch (DdlException e) {
+ LOG.warn("Failed to build analyze job: {}",
+ e.getMessage(), e);
+ }
}
+ return analysisInfos;
} finally {
db.readUnlock();
}
-
}
// Each analyze stmt corresponding to an analysis job.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
index aae783ca8b..b29897304a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
@@ -56,14 +56,59 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
if (!StatisticsUtil.statsTblAvailable()) {
return;
}
+<<<<<<< HEAD
if (Config.enable_auto_collect_statistics) {
analyzePeriodically();
+=======
+ if
(!checkAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
+ return;
+ }
+
+ if (!analysisTaskExecutor.idle()) {
+ return;
+ }
+
+ analyzePeriodically();
+ if (!Config.enable_full_auto_analyze) {
+>>>>>>> 43614cc44f (handle exception properly)
analyzeAutomatically();
}
}
+<<<<<<< HEAD
public void autoAnalyzeStats(DdlStmt ddlStmt) {
// TODO Monitor some DDL statements, and then trigger automatic
analysis tasks
+=======
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private void analyzeAll() {
+ Set<CatalogIf> catalogs =
Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog();
+ for (CatalogIf ctl : catalogs) {
+
+ Collection<DatabaseIf> dbs = ctl.getAllDbs();
+ for (DatabaseIf<TableIf> databaseIf : dbs) {
+ if
(StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName()))
{
+ continue;
+ }
+ AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
+ List<AnalysisInfo> analysisInfos =
analysisManager.buildAnalysisInfosForDB(databaseIf,
+ AnalyzeProperties.DEFAULT_PROP);
+ for (AnalysisInfo analysisInfo : analysisInfos) {
+ analysisInfo = getReAnalyzeRequiredPart(analysisInfo);
+ if (analysisInfo == null) {
+ continue;
+ }
+ try {
+ analysisManager.createSystemAnalysisJob(analysisInfo,
analysisTaskExecutor);
+ } catch (Exception e) {
+ LOG.warn("Failed to create analysis job", e);
+ }
+ }
+ }
+
+ }
+
+ analyzePeriodically();
+>>>>>>> 43614cc44f (handle exception properly)
}
private void analyzePeriodically() {
@@ -116,16 +161,24 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
* @return new job info after check
* @throws Throwable failed to check
*/
+<<<<<<< HEAD
private AnalysisInfo checkAutomaticJobInfo(AnalysisInfo jobInfo) throws
Throwable {
+=======
+ private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
+>>>>>>> 43614cc44f (handle exception properly)
long lastExecTimeInMs = jobInfo.lastExecTimeInMs;
TableIf table = StatisticsUtil
.findTable(jobInfo.catalogName, jobInfo.dbName,
jobInfo.tblName);
- TableStatistic tblStats =
StatisticsRepository.fetchTableLevelStats(table.getId());
+ TableStatistic tblStats = null;
+ try {
+ tblStats =
StatisticsRepository.fetchTableLevelStats(table.getId());
+ } catch (Throwable t) {
+ LOG.warn("Failed to fetch table stats", t);
+ return null;
+ }
if (tblStats == TableStatistic.UNKNOWN) {
- LOG.warn("Failed to automatically analyze statistics, "
- + "no corresponding table statistics for job: {}",
jobInfo.toString());
- throw new DdlException("No corresponding table statistics for
automatic job.");
+ return jobInfo;
}
if (!needReanalyzeTable(table, tblStats)) {
@@ -156,7 +209,7 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
}
private void checkAnalyzedPartitions(TableIf table, Set<String>
statsPartitions,
- Set<String> needRunPartitions, long lastExecTimeInMs) throws
DdlException {
+ Set<String> needRunPartitions, long lastExecTimeInMs) {
for (String statsPartition : statsPartitions) {
Partition partition = table.getPartition(statsPartition);
if (partition == null) {
@@ -165,12 +218,17 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
needRunPartitions.add(statsPartition);
continue;
}
- TableStatistic partitionStats = StatisticsRepository
+ TableStatistic partitionStats = null;
+ try {
+ partitionStats = StatisticsRepository
.fetchTableLevelOfPartStats(partition.getId());
- if (partitionStats == TableStatistic.UNKNOWN) {
+ } catch (DdlException e) {
+ LOG.warn("Failed to fetch part stats", e);
continue;
}
- if (needReanalyzePartition(lastExecTimeInMs, partition,
partitionStats)) {
+
+ if (needReanalyzePartition(lastExecTimeInMs, partition,
partitionStats)
+ || partitionStats == TableStatistic.UNKNOWN) {
needRunPartitions.add(partition.getName());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
