This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 821322c4e22 [branch-3.0](show profile) show query/load profile on
branch-3.0 (#46615)
821322c4e22 is described below
commit 821322c4e22d3c19b0937e3afbe601985ba7fb0b
Author: zhiqiang <[email protected]>
AuthorDate: Thu Jan 9 22:21:49 2025 +0800
[branch-3.0](show profile) show query/load profile on branch-3.0 (#46615)
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 +-
fe/fe-core/src/main/cup/sql_parser.cup | 17 ++++--
.../apache/doris/analysis/ShowLoadProfileStmt.java | 70 ++++++++++++++++-----
.../doris/analysis/ShowQueryProfileStmt.java | 71 +++++++++++++++++-----
.../org/apache/doris/common/profile/Profile.java | 10 +--
.../doris/common/profile/ProfileManager.java | 35 +++++++++--
.../java/org/apache/doris/qe/ShowExecutor.java | 22 +++----
.../suites/query_profile/show_profile.groovy | 60 ++++++++++++++++++
8 files changed, 229 insertions(+), 60 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 53e79f0bb28..177f494905f 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -311,8 +311,8 @@ unsupportedShowStatement
(FROM |IN) tableName=multipartIdentifier
((FROM | IN) database=multipartIdentifier)?
#showIndex
| SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere?
#showTransaction
- | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL
#showQueryProfile
- | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL
#showLoadProfile
+ | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL? limitClause?
#showQueryProfile
+ | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL? limitClause?
#showLoadProfile
| SHOW CACHE HOTSPOT tablePath=STRING_LITERAL
#showCacheHotSpot
| SHOW ENCRYPTKEYS ((FROM | IN) database=multipartIdentifier)? wildWhere?
#showEncryptKeys
| SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?
#showSyncJob
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 07219e6f756..c1c5b0f4fb5 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -858,6 +858,7 @@ nonterminal String opt_job_ends;
nonterminal String job_at_time;
nonterminal ColocateGroupName colocate_group_name;
nonterminal TableScanParams opt_scan_params_ref;
+nonterminal String opt_profile_path;
nonterminal LoadTask.MergeType opt_merge_type, opt_with_merge_type;
@@ -1546,6 +1547,14 @@ quantity ::=
:}
;
+opt_profile_path ::=
+ /* empty */
+ | STRING_LITERAL:queryIdPath
+ {:
+ RESULT = queryIdPath;
+ :}
+ ;
+
opt_user ::=
/* empty */
| KW_FOR user:user
@@ -4646,13 +4655,13 @@ show_param ::=
{:
RESULT = new ShowTransactionStmt(dbName, parser.where);
:}
- | KW_QUERY KW_PROFILE STRING_LITERAL:queryIdPath
+ | KW_QUERY KW_PROFILE opt_profile_path:queryIdPath limit_clause:limitClause
{:
- RESULT = new ShowQueryProfileStmt(queryIdPath);
+ RESULT = new ShowQueryProfileStmt(queryIdPath, limitClause);
:}
- | KW_LOAD KW_PROFILE STRING_LITERAL:loadIdPath
+ | KW_LOAD KW_PROFILE opt_profile_path:loadIdPath limit_clause:limitClause
{:
- RESULT = new ShowLoadProfileStmt(loadIdPath);
+ RESULT = new ShowLoadProfileStmt(loadIdPath, limitClause);
:}
| KW_CACHE KW_HOTSPOT STRING_LITERAL:tablePath
{:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java
index 78437cec796..b4245e4e9f0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java
@@ -17,32 +17,70 @@
package org.apache.doris.analysis;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.UserException;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.qe.ShowResultSetMetaData;
-// deprecated stmt, use will be guided to a specific url to get profile from
-// web browser
+import com.google.common.base.Strings;
+
+// For stmt like:
+// show load profile "/" limit 10;
public class ShowLoadProfileStmt extends ShowStmt implements
NotFallbackInParser {
- private String loadIdPath;
+ // This should be same as ProfileManager.PROFILE_HEADERS
+ public static final ShowResultSetMetaData META_DATA_QUERY_IDS;
+ private static final long DefaultLimit = 20;
+
+ static {
+ ShowResultSetMetaData.Builder builder =
ShowResultSetMetaData.builder();
+ for (String key : SummaryProfile.SUMMARY_KEYS) {
+ if (key.equals(SummaryProfile.DISTRIBUTED_PLAN)) {
+ continue;
+ }
+ builder.addColumn(new Column(key, ScalarType.createStringType()));
+ }
+ META_DATA_QUERY_IDS = builder.build();
+ }
- public ShowLoadProfileStmt(String path) {
- this.loadIdPath = path;
+ public ShowLoadProfileStmt(String useless, LimitElement limit) {
+ this.path = useless;
+ this.limitElement = limit;
}
+ private String path;
+ private final LimitElement limitElement;
+
@Override
- public void analyze(Analyzer analyzer) throws UserException {
- String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
- int httpPort = Config.http_port;
- String terminalMsg = String.format(
- "try visit http://%s:%d/QueryProfile/%s, show query/load
profile syntax is a deprecated feature",
- selfHost, httpPort, this.loadIdPath);
- throw new UserException(terminalMsg);
+ public String toSql() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SHOW LOAD PROFILE");
+ if (!Strings.isNullOrEmpty(path)) {
+ sb.append(" ");
+ sb.append(path);
+ }
+
+ if (limitElement != null && limitElement.getLimit() != -1) {
+ sb.append(" LIMIT ").append(getLimit());
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toSql();
}
@Override
public ShowResultSetMetaData getMetaData() {
- return null;
+ return META_DATA_QUERY_IDS;
+ }
+
+ public long getLimit() {
+ if (limitElement == null) {
+ return DefaultLimit;
+ } else {
+ return limitElement.getLimit() == -1 ? DefaultLimit :
limitElement.getLimit();
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
index ad664652d53..296cb1102fc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
@@ -17,32 +17,71 @@
package org.apache.doris.analysis;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.UserException;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.qe.ShowResultSetMetaData;
-// deprecated stmt, use will be guided to a specific url to get profile from
-// web browser
+import com.google.common.base.Strings;
+
+
+// For stmt like:
+// show query profile "/"; # list all saving query ids
public class ShowQueryProfileStmt extends ShowStmt implements
NotFallbackInParser {
- private String queryIdPath;
+ // This should be same as ProfileManager.PROFILE_HEADERS
+ public static final ShowResultSetMetaData META_DATA_QUERY_IDS;
+ private static final long DefaultLimit = 20;
+
+ static {
+ ShowResultSetMetaData.Builder builder =
ShowResultSetMetaData.builder();
+ for (String key : SummaryProfile.SUMMARY_KEYS) {
+ if (key.equals(SummaryProfile.DISTRIBUTED_PLAN)) {
+ continue;
+ }
+ builder.addColumn(new Column(key, ScalarType.createStringType()));
+ }
+ META_DATA_QUERY_IDS = builder.build();
+ }
+
+ public ShowQueryProfileStmt(String useless, LimitElement limit) {
+ this.path = useless;
+ this.limitElement = limit;
+ }
- public ShowQueryProfileStmt(String queryIdPath) {
- this.queryIdPath = queryIdPath;
+ private final String path;
+ private final LimitElement limitElement;
+
+ @Override
+ public String toSql() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SHOW QUERY PROFILE");
+ if (!Strings.isNullOrEmpty(path)) {
+ sb.append(" ");
+ sb.append(path);
+ }
+
+ if (limitElement != null && limitElement.getLimit() != -1) {
+ sb.append(" LIMIT ").append(getLimit());
+ }
+
+ return sb.toString();
}
@Override
- public void analyze(Analyzer analyzer) throws UserException {
- String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
- int httpPort = Config.http_port;
- String terminalMsg = String.format(
- "try visit http://%s:%d/QueryProfile/%s, show query/load
profile syntax is a deprecated feature",
- selfHost, httpPort, this.queryIdPath);
- throw new UserException(terminalMsg);
+ public String toString() {
+ return toSql();
}
@Override
public ShowResultSetMetaData getMetaData() {
- return null;
+ return META_DATA_QUERY_IDS;
+ }
+
+ public long getLimit() {
+ if (limitElement == null) {
+ return DefaultLimit;
+ } else {
+ return limitElement.getLimit() == -1 ? DefaultLimit :
limitElement.getLimit();
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
index b52179c48ee..9b54695f225 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
@@ -81,22 +81,22 @@ public class Profile {
// profile file name format: time_id
private static final String SEPERATOR = "_";
- // id will be assgined to id of SummaryProfile.
- // For broker load, its SummaryPRofile id is a string representation of a
long integer,
+ // id will be assigned to id of SummaryProfile.
+ // For broker load, its SummaryProfile id is a string representation of a
long integer,
// for others, it is queryID
private String id = "";
// summaryProfile will be serialized to storage as JSON, and we can
recover it from storage
// recover of SummaryProfile is important, because it contains the meta
information of the profile
// we need it to construct memory index for profile retrieving.
private SummaryProfile summaryProfile = new SummaryProfile();
- // executionProfiles will be stored to storage as text, when geting
profile content, we will read
+ // executionProfiles will be stored to storage as text, when getting
profile content, we will read
// from storage directly.
private List<ExecutionProfile> executionProfiles = Lists.newArrayList();
// profileStoragePath will only be assigned when:
// 1. profile is stored to storage
// 2. or profile is loaded from storage
private String profileStoragePath = "";
- // isQueryFinished means the coordinator or stmtexecutor is finished.
+ // isQueryFinished means the coordinator or stmt executor is finished.
// does not mean the profile report has finished, since the report is
async.
// finish of collection of profile is marked by isCompleted of
ExecutionProfiles.
private boolean isQueryFinished = false;
@@ -274,7 +274,7 @@ public class Profile {
}
// This API will also add the profile to ProfileManager, so that we could
get the profile from ProfileManager.
- // isFinished ONLY means the coordinator or stmtexecutor is finished.
+ // isFinished ONLY means the coordinator or stmt executor is finished.
public synchronized void updateSummary(Map<String, String> summaryInfo,
boolean isFinished,
Planner planner) {
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
index 981d025792c..ef9f2f7bbaf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
@@ -120,8 +120,8 @@ public class ProfileManager extends MasterDaemon {
}
}
- // this variable is assgiened to true the first time the profile is loaded
from storage
- // no futher write operaiton, so no data race
+ // this variable is assigned to true the first time the profile is loaded
from storage
+ // no further write operation, so no data race
boolean isProfileLoaded = false;
// only protect queryIdDeque; queryIdToProfileMap is concurrent, no need
to protect
@@ -129,10 +129,10 @@ public class ProfileManager extends MasterDaemon {
private ReadLock readLock;
private WriteLock writeLock;
- // profile id is long string for brocker load
+ // profile id is long string for broker load
// is TUniqueId for others.
private Map<String, ProfileElement> queryIdToProfileMap;
- // Sometimes one Profile is related with multiple execution
profiles(Brokerload), so that
+ // Sometimes one Profile is related with multiple execution
profiles(Broker-load), so that
// execution profile's query id is not related with Profile's query id.
private Map<TUniqueId, ExecutionProfile> queryIdToExecutionProfiles;
@@ -151,7 +151,7 @@ public class ProfileManager extends MasterDaemon {
return INSTANCE;
}
- // The visiablity of ProfileManager() is package level, so that we can
write ut for it.
+ // The visibility of ProfileManager() is package level, so that we can
write ut for it.
ProfileManager() {
super("profile-manager", Config.profile_manager_gc_interval_seconds *
1000);
lock = new ReentrantReadWriteLock(true);
@@ -942,4 +942,29 @@ public class ProfileManager extends MasterDaemon {
}
}
}
+
+ public List<List<String>> getProfileMetaWithType(ProfileType profileType,
long limit) {
+ List<List<String>> result = Lists.newArrayList();
+ readLock.lock();
+
+ try {
+ PriorityQueue<ProfileElement> queueIdDeque =
getProfileOrderByQueryFinishTimeDesc();
+ while (!queueIdDeque.isEmpty() && limit > 0) {
+ ProfileElement profileElement = queueIdDeque.poll();
+ Map<String, String> infoStrings = profileElement.infoStrings;
+ if
(infoStrings.get(SummaryProfile.TASK_TYPE).equals(profileType.toString())) {
+ List<String> row = Lists.newArrayList();
+ for (String str : SummaryProfile.SUMMARY_KEYS) {
+ row.add(infoStrings.get(str));
+ }
+ result.add(row);
+ limit--;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ return result;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index babfe4e2265..0fda1f4e5c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -186,6 +186,8 @@ import org.apache.doris.common.proc.SchemaChangeProcDir;
import org.apache.doris.common.proc.TabletsProcDir;
import org.apache.doris.common.proc.TrashProcDir;
import org.apache.doris.common.proc.TrashProcNode;
+import org.apache.doris.common.profile.ProfileManager;
+import org.apache.doris.common.profile.ProfileManager.ProfileType;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.LogBuilder;
@@ -2619,21 +2621,17 @@ public class ShowExecutor {
}
private void handleShowQueryProfile() throws AnalysisException {
- String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
- int httpPort = Config.http_port;
- String terminalMsg = String.format(
- "try visit http://%s:%d/QueryProfile, show query/load profile
syntax is a deprecated feature",
- selfHost, httpPort);
- throw new AnalysisException(terminalMsg);
+ ShowQueryProfileStmt showStmt = (ShowQueryProfileStmt) stmt;
+ List<List<String>> rows =
ProfileManager.getInstance().getProfileMetaWithType(
+
ProfileType.QUERY, showStmt.getLimit());
+ resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
private void handleShowLoadProfile() throws AnalysisException {
- String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
- int httpPort = Config.http_port;
- String terminalMsg = String.format(
- "try visit http://%s:%d/QueryProfile, show query/load profile
syntax is a deprecated feature",
- selfHost, httpPort);
- throw new AnalysisException(terminalMsg);
+ ShowLoadProfileStmt showStmt = (ShowLoadProfileStmt) stmt;
+ List<List<String>> rows =
ProfileManager.getInstance().getProfileMetaWithType(
+
ProfileType.LOAD, showStmt.getLimit());
+ resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
private void handleShowCreateRepository() throws AnalysisException {
diff --git a/regression-test/suites/query_profile/show_profile.groovy
b/regression-test/suites/query_profile/show_profile.groovy
new file mode 100644
index 00000000000..e0762f531e5
--- /dev/null
+++ b/regression-test/suites/query_profile/show_profile.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('show_profile') {
+ sql "set enable_profile=true;"
+ sql "drop table if exists show_profile"
+ sql "create table show_profile (id int, name varchar(32)) distributed by
hash(id) buckets 1 properties('replication_num'='1');"
+ sql "insert into show_profile values(1, 'a'), (2, 'b'), (3, 'c');"
+
+ for (int i = 0; i < 10; i++) {
+ sql "select * from show_profile;"
+ }
+
+ boolean executedFailed = false
+ try {
+ sql "show query profile;"
+ sql "show load profile;"
+ } catch (Exception e) {
+ logger.error("show query profile failed ", e)
+ executedFailed = true
+ }
+
+ assertEquals(false, executedFailed)
+
+ try {
+ sql """show query profile "/";"""
+ sql """show load profile "/";"""
+ executedFailed = false
+ } catch (Exception e) {
+ logger.error("show profile failed: {}", e)
+ executedFailed = true
+ }
+
+ assertEquals(false, executedFailed)
+
+ try {
+ sql """show query profile "/" limit 10;"""
+ sql """show load profile "/" limit 10;"""
+ executedFailed = false
+ } catch (Exception e) {
+ logger.error("show profile failed: {}", e)
+ executedFailed = true
+ }
+
+ assertEquals(false, executedFailed)
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
