This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 49cb2886602 branch-3.0: [fix](profile) Fix profile of broker load
#46676 (#46695)
49cb2886602 is described below
commit 49cb2886602c1ea4137bdbceb8762fdc479c8dc2
Author: zhiqiang <[email protected]>
AuthorDate: Thu Jan 9 17:43:58 2025 +0800
branch-3.0: [fix](profile) Fix profile of broker load #46676 (#46695)
cherry pick from #46676
---
.../apache/doris/load/loadv2/BrokerLoadJob.java | 3 ++-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 5 ++++
.../apache/doris/load/loadv2/LoadLoadingTask.java | 6 ++---
.../query_profile/s3_load_profile_test.groovy | 28 +++++++++-------------
4 files changed, 21 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index 872d0c5396a..536a7267fdd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -246,7 +246,8 @@ public class BrokerLoadJob extends BulkLoadJob {
this.jobProfile = new Profile(
true,
Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL,
"3")),
-
Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.AUTO_PROFILE_THRESHOLD_MS,
"500")));
+
Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.AUTO_PROFILE_THRESHOLD_MS,
"-1")));
+ this.jobProfile.getSummaryProfile().setQueryBeginTime();
// TODO: 怎么给这些 load job 设置 profile 记录时间
// this.jobProfile.setId("BrokerLoadJob " + id + ". " + label);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index 3e5742e2b05..4f590bd85a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -113,8 +113,13 @@ public abstract class BulkLoadJob extends LoadJob
implements GsonPostProcessable
if (ConnectContext.get() != null) {
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE,
Long.toString(var.getSqlMode()));
+ sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS,
+
Long.toString(var.getAutoProfileThresholdMs()));
+ sessionVariables.put(SessionVariable.PROFILE_LEVEL,
Long.toString(var.getProfileLevel()));
} else {
sessionVariables.put(SessionVariable.SQL_MODE,
String.valueOf(SqlModeHelper.MODE_DEFAULT));
+ sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS,
Long.toString(-1));
+ sessionVariables.put(SessionVariable.PROFILE_LEVEL,
Long.toString(3));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index 8510c2dc124..fd64c2135a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -145,13 +145,13 @@ public class LoadLoadingTask extends LoadTask {
}
protected void executeOnce() throws Exception {
- final boolean enabelProfile = this.jobProfile != null;
+ final boolean enableProfile = this.jobProfile != null;
// New one query id,
Coordinator curCoordinator =
EnvFactory.getInstance().createCoordinator(callback.getCallbackId(),
loadId, planner.getDescTable(),
planner.getFragments(), planner.getScanNodes(),
planner.getTimezone(), loadZeroTolerance,
- enabelProfile);
- if (enabelProfile) {
+ enableProfile);
+ if (enableProfile) {
this.jobProfile.addExecutionProfile(curCoordinator.getExecutionProfile());
}
curCoordinator.setQueryType(TQueryType.LOAD);
diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy
b/regression-test/suites/query_profile/s3_load_profile_test.groovy
index 2d673a2ec58..6405afdf954 100644
--- a/regression-test/suites/query_profile/s3_load_profile_test.groovy
+++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy
@@ -19,7 +19,7 @@ import groovy.json.JsonSlurper
def getProfile = { id ->
def dst = 'http://' + context.config.feHttpAddress
- def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection()
+ def conn = new URL(dst +
"/api/profile/text/?query_id=$id").openConnection()
conn.setRequestMethod("GET")
def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
(context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
@@ -31,9 +31,9 @@ def getProfile = { id ->
suite('s3_load_profile_test') {
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
- sql "drop table if exists dup_tbl_basic;"
+ sql "drop table if exists s3_load_profile_test_dup_tbl_basic;"
sql """
- CREATE TABLE dup_tbl_basic
+ CREATE TABLE s3_load_profile_test_dup_tbl_basic
(
k00 INT NOT NULL,
k01 DATE NOT NULL,
@@ -100,7 +100,7 @@ PROPERTIES (
);
"""
def loadAttribute =new
LoadAttributes("s3://${getS3BucketName()}/regression/load/data/basic_data.csv",
- "dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "s3_load_profile_test_dup_tbl_basic", "LINES TERMINATED BY
\"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "")
def ak = getS3AK()
@@ -109,6 +109,7 @@ PROPERTIES (
sql "set enable_profile=true;"
def label = "test_s3_load_" + UUID.randomUUID().toString().replace("-",
"_")
+ logger.info("s3_load_profile_test_dup_tbl_basic, label: $label")
loadAttribute.label = label
def prop = loadAttribute.getPropertiesStr()
@@ -167,21 +168,14 @@ PROPERTIES (
assertTrue(false, "load Timeout: $loadAttribute.label")
}
}
-
+ Thread.sleep(5000)
qt_select """ select count(*) from $loadAttribute.dataDesc.tableName """
-
+ logger.info("jobId: " + jobId)
def profileString = getProfile(jobId)
- def profileJson = new JsonSlurper().parseText(profileString)
- assertEquals(0, profileJson.code)
- profileDataString = profileJson.data
- logger.info("profileDataString:" + profileDataString)
- def taskStateIdx =
profileDataString.indexOf("Task State: FINISHED")
- assertFalse(taskStateIdx == -1)
- def executionProfileIdx =
profileDataString.indexOf("Execution Profile")
- assertFalse(executionProfileIdx == -1)
- assertTrue(profileDataString.contains("NumScanners"))
- assertTrue(profileDataString.contains("RowsProduced"))
- assertTrue(profileDataString.contains("RowsRead"))
+ logger.info("profileDataString:" + profileString)
+ assertTrue(profileString.contains("NumScanners"))
+ assertTrue(profileString.contains("RowsProduced"))
+ assertTrue(profileString.contains("RowsRead"))
}
class DataDesc {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
