This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8c3f3ffc2d5 [improvement](log)Support exception report in our
benchmark utils (#40834) (#42445)
8c3f3ffc2d5 is described below
commit 8c3f3ffc2d5551478679ae36134d0aa6fb4ecdc4
Author: cat-with-cat <[email protected]>
AuthorDate: Thu Nov 21 11:16:20 2024 +0800
[improvement](log)Support exception report in our benchmark utils (#40834)
(#42445)
## Proposed changes
Issue Number: close #40834
<!--Describe your changes.-->
```
➜ doris_ bash
/home/cat/doris_/doris/tools/clickbench-tools/run-clickbench-queries.sh
query1,ERROR 1105 (HY000) at line 1: errCode = 2, detailMessage = There is
no scanNode Backend available.[10004: not alive]
,ERROR 1105 (HY000) at line 1: errCode = 2, detailMessage = There is no
scanNode Backend available.[10004: not alive]
,ERROR 1105 (HY000) at line 1: errCode = 2, detailMessage = There is no
scanNode Backend available.[10004: not alive]
```
```
➜ doris_ bash
/home/cat/doris_/doris/tools/ssb-tools/bin/run-ssb-flat-queries.sh
============================================
q1.1 317 112 107 107
q1.2 91 52 52 52
q1.3 198 101 136 101
q2.1 3197 1751 1642 1642
q2.2 Error: Failed to execute query q2.2 (cold run). Output:
ERROR 1105 (HY000) at line 17: RpcException, msg: send fragments failed.
io.grpc.StatusRuntimeException: UNAVAILABLE: io exception, host: 192.168.41.145
q2.3 Error: Failed to execute query q2.3 (cold run). Output:
ERROR 1105 (HY000) at line 17: RpcException, msg: send fragments failed.
io.grpc.StatusRuntimeException: UNAVAILABLE: io exception, host: 192.168.41.145
```
```
➜ doris_ bash /home/cat/doris_/doris/tools/ssb-tools/bin/run-ssb-queries.sh
============================================
q1.1 881 210 231 210
q1.2 1075 153 176 153
q1.3 131 124 143 124
q2.1 5429 1551 1638 1551
q2.2 1785 1618 1615 1615
q2.3 1234 1189 1418 1189
q3.1 Error: Failed to execute query q3.1 (cold run). Output:
ERROR 1105 (HY000) at line 17: RpcException, msg: send fragments failed.
io.grpc.StatusRuntimeException: UNAVAILABLE: io exception, host: 192.168.41.145
q3.2 Error: Failed to execute query q3.2 (cold run). Output:
ERROR 1105 (HY000) at line 17: RpcException, msg: send fragments failed.
io.grpc.StatusRuntimeException: UNAVAILABLE: io exception, host: 192.168.41.145
```
```
➜ doris_ bash
/home/cat/doris_/doris/tools/clickbench-tools/load-clickbench-data.sh
FE_HOST: 127.0.0.1
FE_HTTP_PORT: 8030
USER: root
PASSWORD:
DB: clickbench
start...
Curl failed with the following output:
Error: Failed to execute curl to fetch BE's configuration.
```
```
➜ doris_ bash
/home/cat/doris_/doris/tools/clickbench-tools/run-clickbench-queries.sh
mysql Ver 8.0.39-0ubuntu0.22.04.1 for Linux on x86_64 ((Ubuntu))
This is perl 5, version 34, subversion 0 (v5.34.0) built for
x86_64-linux-gnu-thread-multi
(with 60 registered patches, see perl -V for more detail)
Copyright 1987-2021, Larry Wall
Perl may be copied only under the terms of either the Artistic License or
the
GNU General Public License, which may be found in the Perl 5 source kit.
Complete documentation for Perl, including FAQ lists, should be found on
this system using "man perl" or "perldoc perl". If you have access to the
Internet, point your browser at http://www.perl.org/, the Perl Home Page.
FE_HOST: 127.0.0.1
FE_QUERY_PORT: 9030
USER: root
PASSWORD:
DB: clickbench
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1:9030' (111)
```
---
tools/clickbench-tools/load-clickbench-data.sh | 7 ++++++-
tools/ssb-tools/bin/run-ssb-flat-queries.sh | 18 +++++++++++++++---
tools/ssb-tools/bin/run-ssb-queries.sh | 18 +++++++++++++++---
tools/tpcds-tools/bin/run-tpcds-queries.sh | 18 +++++++++++++++---
tools/tpch-tools/bin/run-tpch-queries.sh | 18 +++++++++++++++---
5 files changed, 66 insertions(+), 13 deletions(-)
diff --git a/tools/clickbench-tools/load-clickbench-data.sh
b/tools/clickbench-tools/load-clickbench-data.sh
index 89cc3d22df4..208bdaff52d 100755
--- a/tools/clickbench-tools/load-clickbench-data.sh
+++ b/tools/clickbench-tools/load-clickbench-data.sh
@@ -110,7 +110,12 @@ function check_doris_conf() {
echo "advise: revise your Doris FE's conf to set
'stream_load_default_timeout_second=3600' or above"
fi
- cv=$(curl "${BE_HOST}:${BE_WEBSERVER_PORT}/varz" 2>/dev/null | grep
'streaming_load_max_mb' | awk -F'=' '{print $2}')
+ if ! output=$(curl -s "${BE_HOST}:${BE_WEBSERVER_PORT}/varz" 2>&1); then
+ printf "Curl failed with the following output:\n%s\n" "${output}" >&2
+ printf "Error: Failed to execute curl to fetch BE's configuration.\n"
>&2
+ exit 1
+ fi
+ cv=$(grep 'streaming_load_max_mb' <<< "${output}" | awk -F'=' '{print $2}')
if (($cv < 16000)); then
echo -e "advise: revise your Doris BE's conf to set
'streaming_load_max_mb=16000' or above and 'flush_thread_num_per_store=5' to
speed up load."
fi
diff --git a/tools/ssb-tools/bin/run-ssb-flat-queries.sh
b/tools/ssb-tools/bin/run-ssb-flat-queries.sh
index ab57bccc728..bbd2c2146dd 100755
--- a/tools/ssb-tools/bin/run-ssb-flat-queries.sh
+++ b/tools/ssb-tools/bin/run-ssb-flat-queries.sh
@@ -120,19 +120,31 @@ for i in '1.1' '1.2' '1.3' '2.1' '2.2' '2.3' '3.1' '3.2'
'3.3' '3.4' '4.1' '4.2'
hot2=0
echo -ne "q${i}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (cold run). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
cold=$((end - start))
echo -ne "${cold}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 1). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot1=$((end - start))
echo -ne "${hot1}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 2). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot2=$((end - start))
echo -ne "${hot2}\t" | tee -a result.csv
diff --git a/tools/ssb-tools/bin/run-ssb-queries.sh
b/tools/ssb-tools/bin/run-ssb-queries.sh
index 58cfb5ef7be..ffde7e3962e 100755
--- a/tools/ssb-tools/bin/run-ssb-queries.sh
+++ b/tools/ssb-tools/bin/run-ssb-queries.sh
@@ -120,19 +120,31 @@ for i in '1.1' '1.2' '1.3' '2.1' '2.2' '2.3' '3.1' '3.2'
'3.3' '3.4' '4.1' '4.2'
hot2=0
echo -ne "q${i}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (cold run). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
cold=$((end - start))
echo -ne "${cold}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 1). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot1=$((end - start))
echo -ne "${hot1}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 2). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot2=$((end - start))
echo -ne "${hot2}\t" | tee -a result.csv
diff --git a/tools/tpcds-tools/bin/run-tpcds-queries.sh
b/tools/tpcds-tools/bin/run-tpcds-queries.sh
index ea2b3dd92a2..1af50eea3e9 100755
--- a/tools/tpcds-tools/bin/run-tpcds-queries.sh
+++ b/tools/tpcds-tools/bin/run-tpcds-queries.sh
@@ -144,19 +144,31 @@ for i in ${query_array[@]}; do
hot2=0
echo -ne "query${i}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" --comments
<"${TPCDS_QUERIES_DIR}"/query"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCDS_QUERIES_DIR}/query${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (cold run). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
cold=$((end - start))
echo -ne "${cold}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" --comments
<"${TPCDS_QUERIES_DIR}"/query"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCDS_QUERIES_DIR}/query${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 1). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot1=$((end - start))
echo -ne "${hot1}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" --comments
<"${TPCDS_QUERIES_DIR}"/query"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCDS_QUERIES_DIR}/query${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 2). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot2=$((end - start))
echo -ne "${hot2}\t" | tee -a result.csv
diff --git a/tools/tpch-tools/bin/run-tpch-queries.sh
b/tools/tpch-tools/bin/run-tpch-queries.sh
index 56abe1799ba..43e5a16bb53 100755
--- a/tools/tpch-tools/bin/run-tpch-queries.sh
+++ b/tools/tpch-tools/bin/run-tpch-queries.sh
@@ -141,19 +141,31 @@ for i in ${query_array[@]}; do
hot2=0
echo -ne "q${i}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${TPCH_QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCH_QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (cold run). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
cold=$((end - start))
echo -ne "${cold}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${TPCH_QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCH_QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 1). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot1=$((end - start))
echo -ne "${hot1}\t" | tee -a result.csv
start=$(date +%s%3N)
- mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}"
--comments <"${TPCH_QUERIES_DIR}"/q"${i}".sql >"${RESULT_DIR}"/result"${i}".out
2>"${RESULT_DIR}"/result"${i}".log
+ if ! output=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" --comments \
+ <"${TPCH_QUERIES_DIR}/q${i}.sql" 2>&1); then
+ printf "Error: Failed to execute query q%s (hot run 2). Output:\n%s\n"
"${i}" "${output}" >&2
+ continue
+ fi
end=$(date +%s%3N)
hot2=$((end - start))
echo -ne "${hot2}\t" | tee -a result.csv
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
