This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ad78f313be [Improvement](statistics) show analysis job info (#16305)
ad78f313be is described below
commit ad78f313be91e5c0aca728d4f54d1f43c785169e
Author: ElvinWei <[email protected]>
AuthorDate: Fri Feb 3 23:21:47 2023 +0800
[Improvement](statistics) show analysis job info (#16305)
Supports query analysis job info.
syntax:
```SQL
SHOW ANALYZE
[TABLE | ID]
[
WHERE
[STATE = ["PENDING"|"RUNNING"|"FINISHED"|"FAILED"]]
]
[ORDER BY ...]
[LIMIT limit];
```
example:
```SQL
SHOW ANALYZE test_table1 WHERE state = 'FINISHED' ORDER BY col_name LIMIT 1;
```
result:
| job_id | catalog_name | db_name | tbl_name | col_name |
job_type | analysis_type | message | last_exec_time_in_ms | state |
schedule_type |
| ------ | ------------ | -------------------- | ----------- | -------- |
-------- | ------------- | ------- | -------------------- | -------- |
------------- |
| 10086 | internal | default_cluster:test | test_table1 | pv |
MANUAL | FULL | | 2023-02-01 09:36:41 | FINISHED |
ONCE |
---
fe/fe-core/src/main/cup/sql_parser.cup | 8 +-
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 165 ++++++++++++---------
.../java/org/apache/doris/qe/ShowExecutor.java | 34 ++++-
.../apache/doris/statistics/AnalysisManager.java | 49 +++++-
4 files changed, 174 insertions(+), 82 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 1873ecf5fe..cf2ae1a843 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -3831,14 +3831,14 @@ show_param ::=
RESULT = new ShowCreateMaterializedViewStmt(mvName, tableName);
:}
/* show analyze job */
- | KW_ANALYZE integer_list:jobIds
- {:
- RESULT = new ShowAnalyzeStmt(jobIds);
- :}
| KW_ANALYZE opt_table_name:tbl opt_wild_where
order_by_clause:orderByClause limit_clause:limitClause
{:
RESULT = new ShowAnalyzeStmt(tbl, parser.where, orderByClause,
limitClause);
:}
+ | KW_ANALYZE INTEGER_LITERAL:jobId opt_wild_where
order_by_clause:orderByClause limit_clause:limitClause
+ {:
+ RESULT = new ShowAnalyzeStmt(jobId, parser.where, orderByClause,
limitClause);
+ :}
| KW_CATALOG KW_RECYCLE KW_BIN opt_wild_where
{:
RESULT = new ShowCatalogRecycleBinStmt(parser.where);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index a153b77ce6..a21bb340bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -18,10 +18,8 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -31,15 +29,14 @@ import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.statistics.AnalysisState;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
import java.util.stream.IntStream;
/**
@@ -49,33 +46,37 @@ import java.util.stream.IntStream;
* [TABLE | ID]
* [
* WHERE
- * [STATE =
["PENDING"|"SCHEDULING"|"RUNNING"|"FINISHED"|"FAILED"|"CANCELLED"]]
+ * [STATE = ["PENDING"|"RUNNING"|"FINISHED"|"FAILED"]]
* ]
* [ORDER BY ...]
- * [LIMIT limit][OFFSET offset];
+ * [LIMIT limit];
*/
public class ShowAnalyzeStmt extends ShowStmt {
private static final String STATE_NAME = "state";
private static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
- .add("id")
- .add("create_time")
- .add("start_time")
- .add("finish_time")
- .add("error_msg")
- .add("scope")
- .add("progress")
+ .add("job_id")
+ .add("catalog_name")
+ .add("db_name")
+ .add("tbl_name")
+ .add("col_name")
+ .add("job_type")
+ .add("analysis_type")
+ .add("message")
+ .add("last_exec_time_in_ms")
.add("state")
+ .add("schedule_type")
.build();
- private List<Long> jobIds;
+ private Long jobId;
private TableName dbTableName;
private Expr whereClause;
private LimitElement limitElement;
private List<OrderByElement> orderByElements;
// after analyzed
- private long dbId;
- private final Set<Long> tblIds = Sets.newHashSet();
+ private String catalogName;
+ private String dbName;
+ private String tblName;
private String stateValue;
private ArrayList<OrderByPair> orderByPairs;
@@ -83,10 +84,6 @@ public class ShowAnalyzeStmt extends ShowStmt {
public ShowAnalyzeStmt() {
}
- public ShowAnalyzeStmt(List<Long> jobIds) {
- this.jobIds = jobIds;
- }
-
public ShowAnalyzeStmt(TableName dbTableName,
Expr whereClause,
List<OrderByElement> orderByElements,
@@ -97,44 +94,91 @@ public class ShowAnalyzeStmt extends ShowStmt {
this.limitElement = limitElement;
}
- public List<Long> getJobIds() {
- return jobIds;
+ public ShowAnalyzeStmt(Long jobId,
+ Expr whereClause,
+ List<OrderByElement> orderByElements,
+ LimitElement limitElement) {
+ this.jobId = jobId;
+ this.dbTableName = null;
+ this.whereClause = whereClause;
+ this.orderByElements = orderByElements;
+ this.limitElement = limitElement;
+ }
+
+ public ImmutableList<String> getTitleNames() {
+ return TITLE_NAMES;
+ }
+
+ public Long getJobId() {
+ return jobId;
+ }
+
+ public String getCatalogName() {
+ Preconditions.checkArgument(isAnalyzed(),
+ "The catalogName must be obtained after the parsing is
complete");
+ return catalogName;
}
- public long getDbId() {
+ public String getDbName() {
Preconditions.checkArgument(isAnalyzed(),
- "The dbId must be obtained after the parsing is complete");
- return dbId;
+ "The dbName must be obtained after the parsing is complete");
+ return dbName;
}
- public Set<Long> getTblIds() {
+ public String getTblName() {
Preconditions.checkArgument(isAnalyzed(),
- "The dbId must be obtained after the parsing is complete");
- return tblIds;
+ "The tblName must be obtained after the parsing is complete");
+ return tblName;
}
public String getStateValue() {
Preconditions.checkArgument(isAnalyzed(),
- "The tbl name must be obtained after the parsing is complete");
+ "The stateValue must be obtained after the parsing is
complete");
return stateValue;
}
public ArrayList<OrderByPair> getOrderByPairs() {
Preconditions.checkArgument(isAnalyzed(),
- "The tbl name must be obtained after the parsing is complete");
+ "The orderByPairs must be obtained after the parsing is
complete");
return orderByPairs;
}
- public long getLimit() {
- if (limitElement != null && limitElement.hasLimit()) {
- return limitElement.getLimit();
+ public String getWhereClause() {
+ Preconditions.checkArgument(isAnalyzed(),
+ "The whereClause must be obtained after the parsing is
complete");
+
+ StringBuilder clauseBuilder = new StringBuilder();
+
+ if (jobId != null) {
+ clauseBuilder.append("job_Id = ").append(jobId);
}
- return -1L;
+
+ if (!Strings.isNullOrEmpty(catalogName)) {
+ clauseBuilder.append(clauseBuilder.length() > 0 ? " AND " : "")
+ .append("catalog_name =
\"").append(catalogName).append("\"");
+ }
+
+ if (!Strings.isNullOrEmpty(dbName)) {
+ clauseBuilder.append(clauseBuilder.length() > 0 ? " AND " : "")
+ .append("db_name = \"").append(dbName).append("\"");
+ }
+
+ if (!Strings.isNullOrEmpty(tblName)) {
+ clauseBuilder.append(clauseBuilder.length() > 0 ? " AND " : "")
+ .append("tbl_name = \"").append(tblName).append("\"");
+ }
+
+ if (!Strings.isNullOrEmpty(stateValue)) {
+ clauseBuilder.append(clauseBuilder.length() > 0 ? " AND " : "")
+ .append("state = \"").append(stateValue).append("\"");
+ }
+
+ return clauseBuilder.toString();
}
- public long getOffset() {
- if (limitElement != null && limitElement.hasOffset()) {
- return limitElement.getOffset();
+ public long getLimit() {
+ if (limitElement != null && limitElement.hasLimit()) {
+ return limitElement.getLimit();
}
return -1L;
}
@@ -142,41 +186,15 @@ public class ShowAnalyzeStmt extends ShowStmt {
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
+ catalogName = analyzer.getEnv().getInternalCatalog().getName();
if (dbTableName != null) {
dbTableName.analyze(analyzer);
String dbName = dbTableName.getDb();
String tblName = dbTableName.getTbl();
checkShowAnalyzePriv(dbName, tblName);
-
- Database db =
analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbName);
- Table table = db.getTableOrAnalysisException(tblName);
- dbId = db.getId();
- tblIds.add(table.getId());
- } else {
- // analyze the current default db
- String dbName = analyzer.getDefaultDb();
- if (Strings.isNullOrEmpty(dbName)) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
- }
-
- Database db =
analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbName);
-
- db.readLock();
- try {
- List<Table> tables = db.getTables();
- for (Table table : tables) {
- checkShowAnalyzePriv(dbName, table.getName());
- }
-
- dbId = db.getId();
- for (Table table : tables) {
- long tblId = table.getId();
- tblIds.add(tblId);
- }
- } finally {
- db.readUnlock();
- }
+ this.dbName = dbName;
+ this.tblName = tblName;
}
// analyze where clause if not null
@@ -270,7 +288,7 @@ public class ShowAnalyzeStmt extends ShowStmt {
stateValue = value.toUpperCase();
try {
- // support it later
+ AnalysisState.valueOf(stateValue);
} catch (Exception e) {
valid = false;
}
@@ -295,6 +313,12 @@ public class ShowAnalyzeStmt extends ShowStmt {
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("SHOW ANALYZE");
+
+ if (jobId != null) {
+ sb.append(" ");
+ sb.append(jobId);
+ }
+
if (dbTableName != null) {
sb.append(" ");
sb.append(dbTableName.toSql());
@@ -326,13 +350,6 @@ public class ShowAnalyzeStmt extends ShowStmt {
sb.append(getLimit());
}
- if (getOffset() != -1L) {
- sb.append(" ");
- sb.append("OFFSET");
- sb.append(" ");
- sb.append(getOffset());
- }
-
return sb.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index ea4b384fb2..ddc0c12b37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2308,9 +2308,39 @@ public class ShowExecutor {
resultSet =
Env.getCurrentEnv().getCatalogMgr().showCreateCatalog(showStmt);
}
+ private void handleShowAnalyze() {
+ ShowAnalyzeStmt showStmt = (ShowAnalyzeStmt) stmt;
- private void handleShowAnalyze() throws AnalysisException {
- // TODO: Support later
+ List<List<Comparable>> results;
+ List<List<String>> resultRows = Lists.newArrayList();
+
+ try {
+ results = Env.getCurrentEnv().getAnalysisManager()
+ .showAnalysisJob(showStmt);
+ } catch (DdlException e) {
+ resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows);
+ return;
+ }
+
+ // order the result
+ ListComparator<List<Comparable>> comparator;
+ List<OrderByPair> orderByPairs = showStmt.getOrderByPairs();
+ if (orderByPairs == null) {
+ // sort by id asc
+ comparator = new ListComparator<>(0);
+ } else {
+ OrderByPair[] orderByPairArr = new
OrderByPair[orderByPairs.size()];
+ comparator = new
ListComparator<>(orderByPairs.toArray(orderByPairArr));
+ }
+ results.sort(comparator);
+
+ // convert to result and return it
+ for (List<Comparable> result : results) {
+ List<String> row =
result.stream().map(Object::toString).collect(Collectors.toList());
+ resultRows.add(row);
+ }
+
+ resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows);
}
private void handleCopyTablet() throws AnalysisException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 28fdd4d450..a8f95d9cd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -18,23 +18,30 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AnalyzeStmt;
+import org.apache.doris.analysis.ShowAnalyzeStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.statistics.AnalysisTaskInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisTaskInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisTaskInfo.JobType;
import org.apache.doris.statistics.AnalysisTaskInfo.ScheduleType;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -51,6 +58,14 @@ public class AnalysisManager {
+ FeConstants.INTERNAL_DB_NAME + "." +
StatisticConstants.ANALYSIS_JOB_TABLE + " "
+ "SET state = '${jobState}' ${message} ${updateExecTime} WHERE
job_id = ${jobId}";
+ private static final String SHOW_JOB_STATE_SQL_TEMPLATE = "SELECT "
+ + "job_id, catalog_name, db_name, tbl_name, col_name, job_type, "
+ + "analysis_type, message, last_exec_time_in_ms, state,
schedule_type "
+ + "FROM " + FeConstants.INTERNAL_DB_NAME + "." +
StatisticConstants.ANALYSIS_JOB_TABLE;
+
+ // The time field that needs to be displayed
+ private static final String LAST_EXEC_TIME_IN_MS = "last_exec_time_in_ms";
+
private final ConcurrentMap<Long, Map<Long, AnalysisTaskInfo>>
analysisJobIdToTaskMap;
private StatisticsCache statisticsCache;
@@ -65,6 +80,10 @@ public class AnalysisManager {
taskExecutor.start();
}
+ public StatisticsCache getStatisticsCache() {
+ return statisticsCache;
+ }
+
public void createAnalysisJob(AnalyzeStmt analyzeStmt) {
String catalogName = analyzeStmt.getCatalogName();
String db = analyzeStmt.getDBName();
@@ -142,7 +161,33 @@ public class AnalysisManager {
}
}
- public StatisticsCache getStatisticsCache() {
- return statisticsCache;
+ public List<List<Comparable>> showAnalysisJob(ShowAnalyzeStmt stmt) throws
DdlException {
+ String whereClause = stmt.getWhereClause();
+ long limit = stmt.getLimit();
+ String executeSql = SHOW_JOB_STATE_SQL_TEMPLATE
+ + (whereClause.isEmpty() ? "" : " WHERE " + whereClause)
+ + (limit == -1L ? "" : " LIMIT " + limit);
+
+ List<List<Comparable>> results = Lists.newArrayList();
+ ImmutableList<String> titleNames = stmt.getTitleNames();
+ List<ResultRow> resultRows =
StatisticsUtil.execStatisticQuery(executeSql);
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd
HH:mm:ss");
+
+ for (ResultRow resultRow : resultRows) {
+ List<Comparable> result = Lists.newArrayList();
+ for (String column : titleNames) {
+ String value = resultRow.getColumnValue(column);
+ if (LAST_EXEC_TIME_IN_MS.equals(column)) {
+ long timeMillis = Long.parseLong(value);
+ value = dateFormat.format(new Date(timeMillis));
+ }
+ result.add(value);
+ }
+ results.add(result);
+ }
+
+ return results;
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
