This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d0216d6706 [chore](test) add log to find bug of unstable test (#37982)
5d0216d6706 is described below
commit 5d0216d67061e50ba41a4225951b54a46f77ba19
Author: 924060929 <[email protected]>
AuthorDate: Thu Jul 18 17:20:59 2024 +0800
[chore](test) add log to find bug of unstable test (#37982)
add log to find bug of unstable test
support profile action to fetch profile
---
.../doris/regression/action/HttpCliAction.groovy | 38 ++++++-
.../doris/regression/action/ProfileAction.groovy | 122 +++++++++++++++++++++
.../org/apache/doris/regression/suite/Suite.groovy | 5 +
.../framework/src/main/groovy/suite.gdsl | 4 +-
.../suites/query_p0/cache/sql_cache.groovy | 86 ++++++++++-----
5 files changed, 223 insertions(+), 32 deletions(-)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/HttpCliAction.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/HttpCliAction.groovy
index 8aaf7ef4883..366905b31f5 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/HttpCliAction.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/HttpCliAction.groovy
@@ -17,6 +17,7 @@
package org.apache.doris.regression.action
+import com.google.common.collect.Maps
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.FromString
import groovy.util.logging.Slf4j
@@ -37,7 +38,9 @@ class HttpCliAction implements SuiteAction {
private String body
private String result
private String op
+ private Map<String, String> headers = Maps.newLinkedHashMap()
private Closure check
+ private boolean printResponse = true
SuiteContext context
HttpCliAction(SuiteContext context) {
@@ -60,6 +63,17 @@ class HttpCliAction implements SuiteAction {
this.uri = uri
}
+ void header(String key, String value) {
+ this.headers.put(key, value)
+ }
+
+ void basicAuthorization(String user, String password) {
+ String credentials = user + ":" + (password.is(null) ? "" : password)
+ String encodedCredentials =
Base64.getEncoder().encodeToString(credentials.getBytes())
+ String headerValue = "Basic " + encodedCredentials;
+ headers.put("Authorization", headerValue)
+ }
+
void body(Closure<String> bodySupplier) {
this.body = bodySupplier.call()
}
@@ -80,6 +94,10 @@ class HttpCliAction implements SuiteAction {
this.result = result
}
+ void printResponse(boolean printResponse) {
+ this.printResponse = printResponse
+ }
+
@Override
void run() {
try {
@@ -91,17 +109,25 @@ class HttpCliAction implements SuiteAction {
if (op == "get") {
HttpGet httpGet = new HttpGet(uri)
+ for (final def header in headers.entrySet()) {
+ httpGet.setHeader(header.getKey(), header.getValue())
+ }
client.execute(httpGet).withCloseable { resp ->
resp.withCloseable {
String respJson =
EntityUtils.toString(resp.getEntity())
def respCode = resp.getStatusLine().getStatusCode()
- log.info("respCode: ${respCode}, respJson:
${respJson}")
+ if (printResponse) {
+ log.info("respCode: ${respCode}, respJson:
${respJson}")
+ }
return new ActionResult(respCode, respJson)
}
}
} else {
HttpPost httpPost = new HttpPost(uri)
+ for (final def header in headers.entrySet()) {
+ httpPost.setHeader(header.getKey(), header.getValue())
+ }
StringEntity requestEntity = new StringEntity(
body,
ContentType.APPLICATION_JSON);
@@ -111,14 +137,18 @@ class HttpCliAction implements SuiteAction {
resp.withCloseable {
String respJson =
EntityUtils.toString(resp.getEntity())
def respCode = resp.getStatusLine().getStatusCode()
- log.info("respCode: ${respCode}, respJson:
${respJson}")
+ if (printResponse) {
+ log.info("respCode: ${respCode}, respJson:
${respJson}")
+ }
return new ActionResult(respCode, respJson)
}
}
}
}
- log.info("result:${result}".toString())
- log.info("this.result:${this.result}".toString())
+ if (printResponse) {
+ log.info("result:${result}".toString())
+ log.info("this.result:${this.result}".toString())
+ }
if (check != null) {
check.call(result.respCode, result.body)
} else {
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
new file mode 100644
index 00000000000..b019e6c24aa
--- /dev/null
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.action
+
+import groovy.json.JsonSlurper
+import groovy.transform.stc.ClosureParams
+import groovy.transform.stc.FromString
+import groovy.util.logging.Slf4j
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.util.JdbcUtils
+
+@Slf4j
+class ProfileAction implements SuiteAction {
+ private String tag
+ private Runnable runCallback
+ private Closure<String> check
+ private SuiteContext context
+
+ ProfileAction(SuiteContext context, String tag) {
+ this.context = context
+ this.tag = Objects.requireNonNull(tag, "tag can not be null")
+ }
+
+ void run(@ClosureParams(value = FromString, options = []) Runnable run) {
+ runCallback = run
+ }
+
+ void check(
+ @ClosureParams(value = FromString, options = ["String, Throwable"])
Closure check) {
+ this.check = check
+ }
+
+ @Override
+ void run() {
+ if (runCallback.is(null)) {
+ throw new IllegalStateException("Missing tag")
+ }
+ if (check.is(null)) {
+ throw new IllegalStateException("Missing check")
+ }
+ def conn = context.getConnection()
+ try {
+ JdbcUtils.executeToList(conn, "set enable_profile=true")
+
+ Throwable exception = null
+ try {
+ this.runCallback.run()
+ } catch (Throwable t) {
+ exception = t
+ }
+
+ def httpCli = new HttpCliAction(context)
+ httpCli.endpoint(context.config.feHttpAddress)
+ httpCli.uri("/rest/v1/query_profile")
+ httpCli.op("get")
+ httpCli.printResponse(false)
+
+ if (context.config.fetchRunMode()) {
+ httpCli.basicAuthorization(context.config.feCloudHttpUser,
context.config.feCloudHttpPassword)
+ } else {
+ httpCli.basicAuthorization(context.config.feHttpUser,
context.config.feHttpPassword)
+ }
+ httpCli.check { code, body ->
+ if (code != 200) {
+ throw new IllegalStateException("Get profile list failed,
code: ${code}, body:\n${body}")
+ }
+
+ def jsonSlurper = new JsonSlurper()
+ List profileData = jsonSlurper.parseText(body).data.rows
+ for (final def profileItem in profileData) {
+ if (profileItem["Sql Statement"].toString().contains(tag))
{
+ def profileId = profileItem["Profile ID"].toString()
+
+ def profileCli = new HttpCliAction(context)
+ profileCli.endpoint(context.config.feHttpAddress)
+ profileCli.uri("/rest/v1/query_profile/${profileId}")
+ profileCli.op("get")
+ profileCli.printResponse(false)
+
+ if (context.config.fetchRunMode()) {
+
profileCli.basicAuthorization(context.config.feCloudHttpUser,
context.config.feCloudHttpPassword)
+ } else {
+
profileCli.basicAuthorization(context.config.feHttpUser,
context.config.feHttpPassword)
+ }
+ profileCli.check { profileCode, profileResp ->
+ if (profileCode != 200) {
+ throw new IllegalStateException("Get profile
failed, url: ${"/rest/v1/query_profile/${profileId}"}, code: ${profileCode},
body:\n${profileResp}")
+ }
+
+ def jsonSlurper2 = new JsonSlurper()
+ def profileText =
jsonSlurper2.parseText(profileResp).data
+ profileText = profileText.replace(" ", " ")
+ profileText = profileText.replace("</br>", "\n")
+ this.check(profileText, exception)
+ }
+ profileCli.run()
+
+ break
+ }
+ }
+ }
+ httpCli.run()
+ } finally {
+ JdbcUtils.executeToList(conn, "set enable_profile=false")
+ }
+ }
+}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 50d430b6c6d..1685b1e4d46 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList
import org.apache.commons.lang3.ObjectUtils
import org.apache.doris.regression.Config
import org.apache.doris.regression.action.BenchmarkAction
+import org.apache.doris.regression.action.ProfileAction
import org.apache.doris.regression.action.WaitForAction
import org.apache.doris.regression.util.DataUtils
import org.apache.doris.regression.util.OutputUtils
@@ -620,6 +621,10 @@ class Suite implements GroovyInterceptable {
}
}
+ void profile(String tag, Closure<String> actionSupplier) {
+ runAction(new ProfileAction(context, tag), actionSupplier)
+ }
+
void createMV(String sql) {
(new CreateMVAction(context, sql)).run()
}
diff --git a/regression-test/framework/src/main/groovy/suite.gdsl
b/regression-test/framework/src/main/groovy/suite.gdsl
index aec43ab51a1..bf8da34c2c6 100644
--- a/regression-test/framework/src/main/groovy/suite.gdsl
+++ b/regression-test/framework/src/main/groovy/suite.gdsl
@@ -50,6 +50,7 @@ bindAction("streamLoad",
"org.apache.doris.regression.action.StreamLoadAction")
bindAction("httpTest", "org.apache.doris.regression.action.HttpCliAction")
bindAction("benchmark", "org.apache.doris.regression.action.BenchmarkAction")
bindAction("waitForSchemaChangeDone",
"org.apache.doris.regression.action.WaitForAction")
+bindAction("profile", "org.apache.doris.regression.action.ProfileAction")
// bind qt_xxx and order_qt_xxx methods
contributor([suiteContext]) {
@@ -81,7 +82,8 @@ contributor([suiteContext]) {
!enclosingCall("explain") &&
!enclosingCall("streamLoad") &&
!enclosingCall("httpTest") &&
- !enclosingCall("waitForSchemaChangeDone"))) {
+ !enclosingCall("waitForSchemaChangeDone") &&
+ !enclosingCall("profile"))) {
// bind other suite method and field
def suiteClass = findClass(suiteClassName)
delegatesTo(suiteClass)
diff --git a/regression-test/suites/query_p0/cache/sql_cache.groovy
b/regression-test/suites/query_p0/cache/sql_cache.groovy
index c38c0800740..48f585f3bd9 100644
--- a/regression-test/suites/query_p0/cache/sql_cache.groovy
+++ b/regression-test/suites/query_p0/cache/sql_cache.groovy
@@ -19,12 +19,20 @@
//
/testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
// and modified by Doris.
+import java.util.stream.Collectors
+
suite("sql_cache") {
// TODO: regression-test does not support check query profile,
// so this suite does not check whether cache is used, :)
def tableName = "test_sql_cache"
sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' =
'0')"
+ def variables = sql "show variables"
+ def variableString = variables.stream()
+ .map { it.toString() }
+ .collect(Collectors.joining("\n"))
+ logger.info("Variables:\n${variableString}")
+
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName} (
@@ -183,33 +191,57 @@ suite("sql_cache") {
sql 'set default_order_by_limit = 2'
sql 'set sql_select_limit = 1'
- qt_sql_cache8 """
- select
- k1,
- sum(k2) as total_pv
- from
- ${tableName}
- where
- k1 between '2022-05-28' and '2022-06-30'
- group by
- k1
- order by
- k1;
- """
-
- qt_sql_cache9 """
- select
- k1,
- sum(k2) as total_pv
- from
- ${tableName}
- where
- k1 between '2022-05-28' and '2022-06-30'
- group by
- k1
- order by
- k1;
- """
+ profile("sql_cache8") {
+ run {
+ qt_sql_cache8 """
+ -- sql_cache8
+ select
+ k1,
+ sum(k2) as total_pv
+ from
+ ${tableName}
+ where
+ k1 between '2022-05-28' and '2022-06-30'
+ group by
+ k1
+ order by
+ k1;
+ """
+ }
+
+ check { profileString, exception ->
+ if (!exception.is(null)) {
+ logger.error("Profile failed, profile
result:\n${profileString}", exception)
+ throw exception
+ }
+ }
+ }
+
+ profile("sql_cache9") {
+ run {
+ qt_sql_cache9 """
+ -- sql_cache9
+ select
+ k1,
+ sum(k2) as total_pv
+ from
+ ${tableName}
+ where
+ k1 between '2022-05-28' and '2022-06-30'
+ group by
+ k1
+ order by
+ k1;
+ """
+ }
+
+ check { profileString, exception ->
+ if (!exception.is(null)) {
+ logger.error("Profile failed, profile
result:\n${profileString}", exception)
+ throw exception
+ }
+ }
+ }
sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' =
'10')"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
