This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ac1ae40b31f [ci](perf) print running pipeline tasks to debug (#36610)
ac1ae40b31f is described below
commit ac1ae40b31ff81533558b20e8e305dec290b3b25
Author: Dongyang Li <[email protected]>
AuthorDate: Mon Jun 24 11:58:40 2024 +0800
[ci](perf) print running pipeline tasks to debug (#36610)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
1. print running pipeline tasks to debug
2. fix the bug of deploy when the previous meta changed PR has run build
on the same machine
---------
Co-authored-by: stephen <[email protected]>
---
regression-test/pipeline/common/doris-utils.sh | 10 ++++++++++
regression-test/pipeline/common/github-utils.sh | 17 +++++++++++++++++
regression-test/pipeline/performance/deploy.sh | 4 ++--
regression-test/pipeline/performance/prepare.sh | 9 +++++++++
regression-test/pipeline/performance/run-tpcds.sh | 5 ++++-
regression-test/pipeline/performance/run-tpch.sh | 5 ++++-
6 files changed, 46 insertions(+), 4 deletions(-)
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index 3f4d2c3653b..a38ec1b2f3d 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -733,3 +733,13 @@ function check_if_need_gcore() {
echo "ERROR: unknown exit_flag ${exit_flag}" && return 1
fi
}
+
+function print_running_pipeline_tasks() {
+ webserver_port=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf
webserver_port)
+ mkdir -p "${DORIS_HOME}"/be/log/
+ echo "------------------------${FUNCNAME[0]}--------------------------"
+ echo "curl -m 10
http://127.0.0.1:${webserver_port}/api/running_pipeline_tasks/30";
+ echo ""
+ curl -m 10
"http://127.0.0.1:${webserver_port}/api/running_pipeline_tasks/30"; 2>&1 | tee
"${DORIS_HOME}"/be/log/running_pipeline_tasks_30
+ echo "------------------------${FUNCNAME[0]}--------------------------"
+}
diff --git a/regression-test/pipeline/common/github-utils.sh
b/regression-test/pipeline/common/github-utils.sh
index 9a3c40d7052..28b1010224c 100644
--- a/regression-test/pipeline/common/github-utils.sh
+++ b/regression-test/pipeline/common/github-utils.sh
@@ -377,3 +377,20 @@ file_changed_performance() {
done
echo "return no need" && return 1
}
+
+file_changed_meta() {
+ local all_files
+ all_files=$(cat all_files)
+ if [[ -z ${all_files} ]]; then echo "Failed to get pr changed files." &&
return 0; fi
+ for af in ${all_files}; do
+ if [[ "${af}" ==
'fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java' ||
+ "${af}" ==
'fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java' ||
+ "${af}" ==
'fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java'
||
+ "${af}" ==
'fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java' ||
+ "${af}" == 'gensrc/thrift/'* ||
+ "${af}" == 'gensrc/proto/'* ]]; then
+ echo "Doris meta changed" && return 0
+ fi
+ done
+ echo "Doris meta not changed" && return 1
+}
diff --git a/regression-test/pipeline/performance/deploy.sh
b/regression-test/pipeline/performance/deploy.sh
index dfcf1ecbabd..9f29fd8d256 100644
--- a/regression-test/pipeline/performance/deploy.sh
+++ b/regression-test/pipeline/performance/deploy.sh
@@ -68,8 +68,8 @@ exit_flag=0
cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf
"${DORIS_HOME}"/fe/conf/
cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf
"${DORIS_HOME}"/be/conf/
target_branch="$(echo "${target_branch}" | sed 's| ||g;s|\.||g;s|-||g')" #
remove space、dot、hyphen from branch name
- sed -i
"s|^meta_dir=/data/doris-meta-\${branch_name}|meta_dir=/data/doris-meta-${target_branch}|g"
"${DORIS_HOME}"/fe/conf/fe_custom.conf
- sed -i
"s|^storage_root_path=/data/doris-storage-\${branch_name}|storage_root_path=/data/doris-storage-${target_branch}|g"
"${DORIS_HOME}"/be/conf/be_custom.conf
+ sed -i
"s|^meta_dir=/data/doris-meta-\${branch_name}|meta_dir=/data/doris-meta-${target_branch}${meta_changed_suffix:-}|g"
"${DORIS_HOME}"/fe/conf/fe_custom.conf
+ sed -i
"s|^storage_root_path=/data/doris-storage-\${branch_name}|storage_root_path=/data/doris-storage-${target_branch}${meta_changed_suffix:-}|g"
"${DORIS_HOME}"/be/conf/be_custom.conf
echo "#### 3. start Doris"
meta_dir=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe_custom.conf
meta_dir)
diff --git a/regression-test/pipeline/performance/prepare.sh
b/regression-test/pipeline/performance/prepare.sh
index 1e5231bd2b4..1f8e22aed09 100644
--- a/regression-test/pipeline/performance/prepare.sh
+++ b/regression-test/pipeline/performance/prepare.sh
@@ -94,6 +94,15 @@ if _get_pr_changed_files "${pr_num_from_trigger}"; then
bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'set' "export skip_pipeline=true"
exit 0
fi
+ if file_changed_meta; then
+ # if PR changed the doris meta file, the next PR deployment on the
same mechine which built this PR will fail.
+ # make a copy of the meta file for the meta changed PR.
+ target_branch="$(echo "${target_branch}" | sed 's|
||g;s|\.||g;s|-||g')" # remove space、dot、hyphen from branch name
+ meta_changed_suffix="_2"
+ rsync -a --delete "/data/doris-meta-${target_branch}/"
"/data/doris-meta-${target_branch}${meta_changed_suffix}"
+ rsync -a --delete "/data/doris-storage-${target_branch}/"
"/data/doris-storage-${target_branch}${meta_changed_suffix}"
+ bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'set' "export meta_changed_suffix=${meta_changed_suffix}"
+ fi
fi
echo "#### 2. check if tpch depending files exist"
diff --git a/regression-test/pipeline/performance/run-tpcds.sh
b/regression-test/pipeline/performance/run-tpcds.sh
index 5c24c7ab9c5..478d822059a 100644
--- a/regression-test/pipeline/performance/run-tpcds.sh
+++ b/regression-test/pipeline/performance/run-tpcds.sh
@@ -132,7 +132,10 @@ exit_flag=0
hot_run_time_threshold=${hot_run_time_threshold_branch20:-260000} #
ms
fi
echo "INFO: cold_run_time_threshold is ${cold_run_time_threshold},
hot_run_time_threshold is ${hot_run_time_threshold}"
- if ! check_tpcds_result
"${teamcity_build_checkoutDir}"/run-tpcds-queries.log; then exit 1; fi
+ if ! check_tpcds_result
"${teamcity_build_checkoutDir}"/run-tpcds-queries.log; then
+ print_running_pipeline_tasks
+ exit 1
+ fi
line_end=$(sed -n '/^Total hot run time/='
"${teamcity_build_checkoutDir}"/run-tpcds-queries.log)
line_begin=$((line_end - 100))
comment_body_summary="$(sed -n "${line_end}p"
"${teamcity_build_checkoutDir}"/run-tpcds-queries.log)"
diff --git a/regression-test/pipeline/performance/run-tpch.sh
b/regression-test/pipeline/performance/run-tpch.sh
index 86ea13b6a0c..36df5d575ce 100644
--- a/regression-test/pipeline/performance/run-tpch.sh
+++ b/regression-test/pipeline/performance/run-tpch.sh
@@ -124,7 +124,10 @@ exit_flag=0
hot_run_time_threshold=${hot_run_time_threshold_branch20:-55000} #
ms
fi
echo "INFO: cold_run_time_threshold is ${cold_run_time_threshold},
hot_run_time_threshold is ${hot_run_time_threshold}"
- if ! check_tpch_result
"${teamcity_build_checkoutDir}"/run-tpch-queries.log; then exit 1; fi
+ if ! check_tpch_result
"${teamcity_build_checkoutDir}"/run-tpch-queries.log; then
+ print_running_pipeline_tasks
+ exit 1
+ fi
line_end=$(sed -n '/^Total hot run time/='
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)
line_begin=$((line_end - 23))
comment_body_summary="$(sed -n "${line_end}p"
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
