This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 726ab9d522b [ci](cloud) align pipeline to check leaks (#53085)
726ab9d522b is described below
commit 726ab9d522be670ca6f67597295183b0dcf2e496
Author: Dongyang Li <[email protected]>
AuthorDate: Fri Jul 25 17:08:45 2025 +0800
[ci](cloud) align pipeline to check leaks (#53085)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: pick #52886 #52725
---
regression-test/pipeline/cloud_p0/clean.sh | 2 +-
regression-test/pipeline/cloud_p0/run.sh | 20 +++--
regression-test/pipeline/common/doris-utils.sh | 100 ++++++++++++++++++++++---
regression-test/pipeline/vault_p0/clean.sh | 3 +-
regression-test/pipeline/vault_p0/run.sh | 20 +++--
5 files changed, 116 insertions(+), 29 deletions(-)
diff --git a/regression-test/pipeline/cloud_p0/clean.sh
b/regression-test/pipeline/cloud_p0/clean.sh
index 2b67ae93495..9af014bb6e3 100644
--- a/regression-test/pipeline/cloud_p0/clean.sh
+++ b/regression-test/pipeline/cloud_p0/clean.sh
@@ -26,5 +26,5 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build
pipline" && exit 0; else
echo "#### Run tpcds test on Doris ####"
DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
-export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace
clean_fdb "cloud_instance_0"
+exit
diff --git a/regression-test/pipeline/cloud_p0/run.sh
b/regression-test/pipeline/cloud_p0/run.sh
index 55f0c419013..6cca8b92af8 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -111,15 +111,19 @@ if print_running_pipeline_tasks; then :; fi
# shellcheck source=/dev/null
source "$(cd "${teamcity_build_checkoutDir}" && bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
-echo "#### 5. check if need backup doris logs"
+check_if_need_gcore "${exit_flag}"
+if stop_doris_grace; then
+ echo "INFO: stop doris grace success."
+else
+ echo "ERROR: stop grace failed." && exit_flag=2
+fi
+if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
+ reporting_build_problem "coredump"
+ print_doris_fe_log
+ print_doris_be_log
+fi
+echo "#### check if need backup doris logs ####"
if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then
- check_if_need_gcore "${exit_flag}"
- if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
- reporting_build_problem "coredump"
- print_doris_fe_log
- print_doris_be_log
- fi
- export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace
if log_file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then
reporting_messages_error "${log_info##*logs.tar.gz to }"
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index 44f075e14e1..06e971201c5 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -222,14 +222,87 @@ function stop_doris() {
function stop_doris_grace() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
- if "${DORIS_HOME}"/be/bin/stop_be.sh --grace &&
"${DORIS_HOME}"/fe/bin/stop_fe.sh --grace; then
- echo "INFO: normally stoped doris --grace"
+ local ret=0
+ local keywords="detected memory leak|undefined-behavior"
+ sudo mkdir -p /tmp/be/bin && cp -rf "${DORIS_HOME}"/be/bin/be.pid
/tmp/be/bin/be.pid
+ if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/be/bin/stop_be.sh --grace; then
+ echo "INFO: doris be stopped gracefully."
+ if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
+ echo "INFO: try to find keywords ${keywords} in be.out"
+ if [[ -f "${DORIS_HOME}"/be/log/be.out ]]; then
+ if grep -E "${keywords}" "${DORIS_HOME}"/be/log/be.out; then
+ echo "##teamcity[buildProblem description='Ubsan or Lsan
fail']"
+ echo "====================================head -n 200
be/log/be.out===================================="
+ head -n 200 "${DORIS_HOME}"/be/log/be.out
+ echo
"================================================================================================="
+ echo "ERROR: found memory leaks or undefined behavior in
be.out" && ret=1
+ else
+ echo "INFO: no memory leaks or undefined behavior found in
be.out"
+ fi
+ else
+ echo "##teamcity[buildProblem description='Stop BE grace
fail']"
+ echo "ERROR: be.out not find, which is not expected" && ret=1
+ fi
+ fi
else
- pgrep -fi doris | xargs kill -9 &>/dev/null
- echo "WARNING: force stoped doris"
+ echo "ERROR: doris be stop grace failed." && ret=1
fi
- if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then bash
"${DORIS_HOME}"/ms/bin/stop.sh --grace; fi
- if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then bash
"${DORIS_HOME}"/recycler/bin/stop.sh --grace; fi
+ if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/fe/bin/stop_fe.sh --grace; then
+ echo "INFO: doris fe stopped gracefully."
+ else
+ echo "ERROR: doris fe stop grace failed." && ret=1
+ fi
+ if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then
+ sudo mkdir -p /tmp/ms/bin && cp -rf
"${DORIS_HOME}"/ms/bin/doris_cloud.pid /tmp/ms/bin/doris_cloud.pid
+ if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/ms/bin/stop.sh --grace; then
+ echo "INFO: doris ms stopped gracefully."
+ if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
+ echo "INFO: try to find keywords ${keywords} in
doris_cloud.out"
+ if [[ -f "${DORIS_HOME}"/ms/log/doris_cloud.out ]]; then
+ if grep -E "${keywords}"
"${DORIS_HOME}"/ms/log/doris_cloud.out; then
+ echo "##teamcity[buildProblem description='Ubsan or
Lsan fail']"
+ echo "====================================head -n 200
ms/log/doris_cloud.out===================================="
+ head -n 200 "${DORIS_HOME}"/ms/log/doris_cloud.out
+ echo
"=========================================================================================================="
+ echo "ERROR: found memory leaks or undefined behavior
in ms/log/doris_cloud.out" && ret=1
+ else
+ echo "INFO: no memory leaks or undefined behavior
found in ms/log/doris_cloud.out"
+ fi
+ else
+ echo "ERROR: ms/log/doris_cloud.out not find, which is not
expected" && ret=1
+ fi
+ fi
+ else
+ echo "##teamcity[buildProblem description='Stop MS grace fail']"
+ echo "ERROR: doris ms stop grace failed." && ret=1
+ fi
+ fi
+ if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then
+ sudo mkdir -p /tmp/recycler/bin && cp -rf
"${DORIS_HOME}"/recycler/bin/doris_cloud.pid /tmp/recycler/bin/doris_cloud.pid
+ if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/recycler/bin/stop.sh --grace; then
+ echo "INFO: doris recycler stopped gracefully."
+ # if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
+ # echo "INFO: try to find keywords ${keywords} in
doris_cloud.out"
+ # if [[ -f "${DORIS_HOME}"/recycler/log/doris_cloud.out ]];
then
+ # if grep -E "${keywords}"
"${DORIS_HOME}"/recycler/log/doris_cloud.out; then
+ # echo "##teamcity[buildProblem description='Ubsan or
Lsan fail']"
+ # echo "=================================head -n 200
recycler/log/doris_cloud.out================================="
+ # head -n 200
"${DORIS_HOME}"/recycler/log/doris_cloud.out
+ # echo
"=========================================================================================================="
+ # echo "ERROR: found memory leaks or undefined
behavior in recycler/log/doris_cloud.out" && ret=1
+ # else
+ # echo "INFO: no memory leaks or undefined behavior
found in recycler/log/doris_cloud.out"
+ # fi
+ # else
+ # echo "ERROR: recycler/log/doris_cloud.out not find,
which is not expected" && ret=1
+ # fi
+ # fi
+ else
+ echo "##teamcity[buildProblem description='Stop RECYCLER grace
fail']"
+ echo "ERROR: doris recycler stop grace failed." && ret=1
+ fi
+ fi
+ return "${ret}"
}
function clean_fdb() {
@@ -299,7 +372,7 @@ deploy_doris_sql_converter() {
fi
}
-function restart_doris() {
+function _restart_doris() {
if stop_doris; then echo; fi
if ! start_doris_fe; then return 1; fi
if ! start_doris_be; then return 1; fi
@@ -319,6 +392,11 @@ function restart_doris() {
sleep 10s
}
+function restart_doris() {
+ # restart BE may block on JVM_MonitorWait() for a long time, here try twice
+ _restart_doris || _restart_doris
+}
+
function check_tpch_table_rows() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
db_name="$1"
@@ -663,9 +741,9 @@ archive_doris_coredump() {
rm -rf "${DORIS_HOME:?}/${archive_dir}"
mkdir -p "${DORIS_HOME}/${archive_dir}"
declare -A pids
- pids['be']="$(cat "${DORIS_HOME}"/be/bin/be.pid)"
- pids['ms']="$(cat "${DORIS_HOME}"/ms/bin/doris_cloud.pid)"
- pids['recycler']="$(cat "${DORIS_HOME}"/recycler/bin/doris_cloud.pid)"
+ pids['be']="$(cat /tmp/be/bin/be.pid)"
+ pids['ms']="$(cat /tmp/ms/bin/doris_cloud.pid)"
+ pids['recycler']="$(cat /tmp/recycler/bin/doris_cloud.pid)"
local has_core=false
for p in "${!pids[@]}"; do
pid="${pids[${p}]}"
@@ -877,7 +955,7 @@ function check_if_need_gcore() {
sleep 10
fi
else
- echo "ERROR: unknown exit_flag ${exit_flag}" && return 1
+ echo "ERROR: exit_flag ${exit_flag} is not 124(timeout), no need to
gcore" && return 1
fi
}
diff --git a/regression-test/pipeline/vault_p0/clean.sh
b/regression-test/pipeline/vault_p0/clean.sh
index 1500bd48bf4..d72767e158a 100644
--- a/regression-test/pipeline/vault_p0/clean.sh
+++ b/regression-test/pipeline/vault_p0/clean.sh
@@ -26,8 +26,9 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build
pipline" && exit 0; else
echo "#### stop doris and clean fdb ####"
DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
-export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace
clean_fdb "cloud_instance_0"
echo "#### docker-compose down ####"
docker-compose down
+
+exit
diff --git a/regression-test/pipeline/vault_p0/run.sh
b/regression-test/pipeline/vault_p0/run.sh
index a292884ae2d..e6e7217a789 100644
--- a/regression-test/pipeline/vault_p0/run.sh
+++ b/regression-test/pipeline/vault_p0/run.sh
@@ -153,15 +153,19 @@ if print_running_pipeline_tasks; then :; fi
# shellcheck source=/dev/null
source "$(cd "${teamcity_build_checkoutDir}" && bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
-echo "#### 5. check if need backup doris logs"
+check_if_need_gcore "${exit_flag}"
+if stop_doris_grace; then
+ echo "INFO: stop doris grace success."
+else
+ echo "ERROR: stop grace failed." && exit_flag=2
+fi
+if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
+ reporting_build_problem "coredump"
+ print_doris_fe_log
+ print_doris_be_log
+fi
+echo "#### check if need backup doris logs ####"
if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then
- check_if_need_gcore "${exit_flag}"
- if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
- reporting_build_problem "coredump"
- print_doris_fe_log
- print_doris_be_log
- fi
- export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace
if log_file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then
reporting_messages_error "${log_info##*logs.tar.gz to }"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
