This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b3b9de1f77b [ci](cloud_p0) pipeline fail if stop grace fail (#52886)
b3b9de1f77b is described below
commit b3b9de1f77bcd8934a80fab04acda4cc27b4c09e
Author: Dongyang Li <[email protected]>
AuthorDate: Wed Jul 9 21:56:56 2025 +0800
[ci](cloud_p0) pipeline fail if stop grace fail (#52886)
---
regression-test/pipeline/cloud_p0/clean.sh | 3 +-
regression-test/pipeline/cloud_p0/run.sh | 20 ++++++-----
regression-test/pipeline/common/doris-utils.sh | 50 +++++++++++++++++---------
regression-test/pipeline/vault_p0/clean.sh | 3 +-
regression-test/pipeline/vault_p0/run.sh | 20 ++++++-----
5 files changed, 60 insertions(+), 36 deletions(-)
diff --git a/regression-test/pipeline/cloud_p0/clean.sh
b/regression-test/pipeline/cloud_p0/clean.sh
index 752c38b5f02..9af014bb6e3 100644
--- a/regression-test/pipeline/cloud_p0/clean.sh
+++ b/regression-test/pipeline/cloud_p0/clean.sh
@@ -26,6 +26,5 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build
pipline" && exit 0; else
echo "#### Run tpcds test on Doris ####"
DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
-if stop_doris_grace; then exit_flag=0; else exit_flag=1; fi
clean_fdb "cloud_instance_0"
-exit "${exit_flag}"
+exit
diff --git a/regression-test/pipeline/cloud_p0/run.sh
b/regression-test/pipeline/cloud_p0/run.sh
index 040e0a7b767..6cca8b92af8 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -111,15 +111,19 @@ if print_running_pipeline_tasks; then :; fi
# shellcheck source=/dev/null
source "$(cd "${teamcity_build_checkoutDir}" && bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
-echo "#### 5. check if need backup doris logs"
+check_if_need_gcore "${exit_flag}"
+if stop_doris_grace; then
+ echo "INFO: stop doris grace success."
+else
+ echo "ERROR: stop grace failed." && exit_flag=2
+fi
+if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
+ reporting_build_problem "coredump"
+ print_doris_fe_log
+ print_doris_be_log
+fi
+echo "#### check if need backup doris logs ####"
if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then
- check_if_need_gcore "${exit_flag}"
- if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
- reporting_build_problem "coredump"
- print_doris_fe_log
- print_doris_be_log
- fi
- stop_doris_grace
if log_file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then
reporting_messages_error "${log_info##*logs.tar.gz to }"
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index 6d90376b592..efafb7742b7 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -220,17 +220,23 @@ function stop_doris_grace() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
local ret=0
local keywords="detected memory leak|undefined-behavior"
+ sudo mkdir -p /tmp/be/bin && cp -rf "${DORIS_HOME}"/be/bin/be.pid
/tmp/be/bin/be.pid
if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/be/bin/stop_be.sh --grace; then
echo "INFO: doris be stopped gracefully."
if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
echo "INFO: try to find keywords ${keywords} in be.out"
if [[ -f "${DORIS_HOME}"/be/log/be.out ]]; then
if grep -E "${keywords}" "${DORIS_HOME}"/be/log/be.out; then
+ echo "##teamcity[buildProblem description='Ubsan or Lsan
fail']"
+ echo "====================================head -n 200
be/log/be.out===================================="
+ head -n 200 "${DORIS_HOME}"/be/log/be.out
+ echo
"================================================================================================="
echo "ERROR: found memory leaks or undefined behavior in
be.out" && ret=1
else
echo "INFO: no memory leaks or undefined behavior found in
be.out"
fi
else
+ echo "##teamcity[buildProblem description='Stop BE grace
fail']"
echo "ERROR: be.out not find, which is not expected" && ret=1
fi
fi
@@ -243,12 +249,17 @@ function stop_doris_grace() {
echo "ERROR: doris fe stop grace failed." && ret=1
fi
if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then
+ sudo mkdir -p /tmp/ms/bin && cp -rf
"${DORIS_HOME}"/ms/bin/doris_cloud.pid /tmp/ms/bin/doris_cloud.pid
if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/ms/bin/stop.sh --grace; then
echo "INFO: doris ms stopped gracefully."
if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
echo "INFO: try to find keywords ${keywords} in
doris_cloud.out"
if [[ -f "${DORIS_HOME}"/ms/log/doris_cloud.out ]]; then
if grep -E "${keywords}"
"${DORIS_HOME}"/ms/log/doris_cloud.out; then
+ echo "##teamcity[buildProblem description='Ubsan or
Lsan fail']"
+ echo "====================================head -n 200
ms/log/doris_cloud.out===================================="
+ head -n 200 "${DORIS_HOME}"/ms/log/doris_cloud.out
+ echo
"=========================================================================================================="
echo "ERROR: found memory leaks or undefined behavior
in ms/log/doris_cloud.out" && ret=1
else
echo "INFO: no memory leaks or undefined behavior
found in ms/log/doris_cloud.out"
@@ -258,25 +269,32 @@ function stop_doris_grace() {
fi
fi
else
+ echo "##teamcity[buildProblem description='Stop MS grace fail']"
echo "ERROR: doris ms stop grace failed." && ret=1
fi
fi
if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then
+ sudo mkdir -p /tmp/recycler/bin && cp -rf
"${DORIS_HOME}"/recycler/bin/doris_cloud.pid /tmp/recycler/bin/doris_cloud.pid
if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash
"${DORIS_HOME}"/recycler/bin/stop.sh --grace; then
echo "INFO: doris recycler stopped gracefully."
- echo "INFO: try to find keywords ${keywords} in doris_cloud.out"
- if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
- if [[ -f "${DORIS_HOME}"/recycler/log/doris_cloud.out ]]; then
- if grep -E "${keywords}"
"${DORIS_HOME}"/recycler/log/doris_cloud.out; then
- echo "ERROR: found memory leaks or undefined behavior
in recycler/log/doris_cloud.out" && ret=1
- else
- echo "INFO: no memory leaks or undefined behavior
found in recycler/log/doris_cloud.out"
- fi
- else
- echo "ERROR: recycler/log/doris_cloud.out not find, which
is not expected" && ret=1
- fi
- fi
+ # if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" &&
"${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then
+ # echo "INFO: try to find keywords ${keywords} in
doris_cloud.out"
+ # if [[ -f "${DORIS_HOME}"/recycler/log/doris_cloud.out ]];
then
+ # if grep -E "${keywords}"
"${DORIS_HOME}"/recycler/log/doris_cloud.out; then
+ # echo "##teamcity[buildProblem description='Ubsan or
Lsan fail']"
+ # echo "=================================head -n 200
recycler/log/doris_cloud.out================================="
+ # head -n 200
"${DORIS_HOME}"/recycler/log/doris_cloud.out
+ # echo
"=========================================================================================================="
+ # echo "ERROR: found memory leaks or undefined
behavior in recycler/log/doris_cloud.out" && ret=1
+ # else
+ # echo "INFO: no memory leaks or undefined behavior
found in recycler/log/doris_cloud.out"
+ # fi
+ # else
+ # echo "ERROR: recycler/log/doris_cloud.out not find,
which is not expected" && ret=1
+ # fi
+ # fi
else
+ echo "##teamcity[buildProblem description='Stop RECYCLER grace
fail']"
echo "ERROR: doris recycler stop grace failed." && ret=1
fi
fi
@@ -719,9 +737,9 @@ archive_doris_coredump() {
rm -rf "${DORIS_HOME:?}/${archive_dir}"
mkdir -p "${DORIS_HOME}/${archive_dir}"
declare -A pids
- pids['be']="$(cat "${DORIS_HOME}"/be/bin/be.pid)"
- pids['ms']="$(cat "${DORIS_HOME}"/ms/bin/doris_cloud.pid)"
- pids['recycler']="$(cat "${DORIS_HOME}"/recycler/bin/doris_cloud.pid)"
+ pids['be']="$(cat /tmp/be/bin/be.pid)"
+ pids['ms']="$(cat /tmp/ms/bin/doris_cloud.pid)"
+ pids['recycler']="$(cat /tmp/recycler/bin/doris_cloud.pid)"
local has_core=false
for p in "${!pids[@]}"; do
pid="${pids[${p}]}"
@@ -933,7 +951,7 @@ function check_if_need_gcore() {
sleep 10
fi
else
- echo "ERROR: unknown exit_flag ${exit_flag}" && return 1
+ echo "ERROR: exit_flag ${exit_flag} is not 124(timeout), no need to
gcore" && return 1
fi
}
diff --git a/regression-test/pipeline/vault_p0/clean.sh
b/regression-test/pipeline/vault_p0/clean.sh
index 4dc0f59df1d..d72767e158a 100644
--- a/regression-test/pipeline/vault_p0/clean.sh
+++ b/regression-test/pipeline/vault_p0/clean.sh
@@ -26,10 +26,9 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build
pipline" && exit 0; else
echo "#### stop doris and clean fdb ####"
DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
-if stop_doris_grace; then exit_flag=0; else exit_flag=1; fi
clean_fdb "cloud_instance_0"
echo "#### docker-compose down ####"
docker-compose down
-exit "${exit_flag}"
+exit
diff --git a/regression-test/pipeline/vault_p0/run.sh
b/regression-test/pipeline/vault_p0/run.sh
index a7492f1416c..db8c20f0545 100644
--- a/regression-test/pipeline/vault_p0/run.sh
+++ b/regression-test/pipeline/vault_p0/run.sh
@@ -148,15 +148,19 @@ if print_running_pipeline_tasks; then :; fi
# shellcheck source=/dev/null
source "$(cd "${teamcity_build_checkoutDir}" && bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
-echo "#### 5. check if need backup doris logs"
+check_if_need_gcore "${exit_flag}"
+if stop_doris_grace; then
+ echo "INFO: stop doris grace success."
+else
+ echo "ERROR: stop grace failed." && exit_flag=2
+fi
+if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
+ reporting_build_problem "coredump"
+ print_doris_fe_log
+ print_doris_be_log
+fi
+echo "#### check if need backup doris logs ####"
if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then
- check_if_need_gcore "${exit_flag}"
- if core_file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
- reporting_build_problem "coredump"
- print_doris_fe_log
- print_doris_be_log
- fi
- stop_doris_grace
if log_file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then
reporting_messages_error "${log_info##*logs.tar.gz to }"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
