This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d459ac239f0 [chore](ci) prepare jdk17 in cloud_p* and performance
(#31685)
d459ac239f0 is described below
commit d459ac239f005253fd32aea7982ab2d2df2fa5da
Author: Dongyang Li <[email protected]>
AuthorDate: Sun Mar 3 23:57:52 2024 +0800
[chore](ci) prepare jdk17 in cloud_p* and performance (#31685)
---
regression-test/pipeline/cloud_p0/deploy.sh | 2 ++
regression-test/pipeline/cloud_p0/prepare.sh | 5 +++--
regression-test/pipeline/cloud_p0/run.sh | 4 +++-
regression-test/pipeline/common/doris-utils.sh | 28 +++++++++++++++++--------
regression-test/pipeline/performance/compile.sh | 6 +++++-
regression-test/pipeline/performance/prepare.sh | 15 +++++++++++--
6 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/regression-test/pipeline/cloud_p0/deploy.sh
b/regression-test/pipeline/cloud_p0/deploy.sh
index 788d5f24457..c65de2b39c6 100644
--- a/regression-test/pipeline/cloud_p0/deploy.sh
+++ b/regression-test/pipeline/cloud_p0/deploy.sh
@@ -69,6 +69,8 @@ exit_flag=0
print_doris_conf
echo "#### 4. start Doris"
+ JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*' | sed
-n '1p')"
+ export JAVA_HOME
if ! start_doris_ms; then exit 1; fi
if ! start_doris_recycler; then exit 1; fi
if ! create_warehouse; then exit 1; fi
diff --git a/regression-test/pipeline/cloud_p0/prepare.sh
b/regression-test/pipeline/cloud_p0/prepare.sh
index 25a7d9f3d74..b468a7b3bd0 100644
--- a/regression-test/pipeline/cloud_p0/prepare.sh
+++ b/regression-test/pipeline/cloud_p0/prepare.sh
@@ -61,7 +61,7 @@ if ${DEBUG:-false}; then
fi
# shellcheck source=/dev/null
-# stop_doris, clean_fdb, install_fdb
+# stop_doris, clean_fdb, install_fdb, install_java
source
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
# shellcheck source=/dev/null
# check_oss_file_exist
@@ -94,7 +94,8 @@ fi
source "$(bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0;
else echo "INFO: no skip"; fi
if [[ "${target_branch}" == "master" ]]; then
- echo "INFO: PR target branch ${target_branch} is in (master)"
+ echo "INFO: PR target branch ${target_branch}"
+ install_java
else
echo "WARNING: PR target branch ${target_branch} is NOT in (master), skip
pipeline."
bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'set' "export skip_pipeline=true"
diff --git a/regression-test/pipeline/cloud_p0/run.sh
b/regression-test/pipeline/cloud_p0/run.sh
index 4cfe066d1d0..68bf3e3da6a 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -36,7 +36,7 @@ if [[ -z "${cos_ak}" || -z "${cos_sk}" ]]; then echo "ERROR:
env cos_ak or cos_s
source "$(bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0;
else echo "INFO: no skip"; fi
-echo "#### Run tpcds test on Doris ####"
+echo "#### Run cloud_p0 test on Doris ####"
DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
exit_flag=0
@@ -51,6 +51,8 @@ run() {
echo "sk='${cos_sk}'"
>>"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
\
"${teamcity_build_checkoutDir}"/regression-test/conf/
+ JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed
-n '1p')"
+ export JAVA_HOME
if "${teamcity_build_checkoutDir}"/run-regression-test.sh \
--teamcity \
--clean \
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index ab2f3a86e54..95b1f7faf80 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -89,14 +89,25 @@ function start_doris_recycler() {
fi
}
-function start_doris_fe() {
- if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+function install_java() {
if ! java -version >/dev/null ||
[[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]];
then
sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
fi
- JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed
-n '1p')"
- export JAVA_HOME
+ # doris master branch use java-17
+ if ! java -version >/dev/null ||
+ [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*')" ]];
then
+ sudo apt update && sudo apt install openjdk-17-jdk -y >/dev/null
+ fi
+}
+
+function start_doris_fe() {
+ if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+ if install_java && [[ -z "${JAVA_HOME}" ]]; then
+ # default to use java-8
+ JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' |
sed -n '1p')"
+ export JAVA_HOME
+ fi
# export
JACOCO_COVERAGE_OPT="-javaagent:/usr/local/jacoco/lib/jacocoagent.jar=excludes=org.apache.doris.thrift:org.apache.doris.proto:org.apache.parquet.format:com.aliyun*:com.amazonaws*:org.apache.hadoop.hive.metastore:org.apache.parquet.format,output=file,append=true,destfile=${DORIS_HOME}/fe/fe_cov.exec"
"${DORIS_HOME}"/fe/bin/start_fe.sh --daemon
@@ -117,12 +128,11 @@ function start_doris_fe() {
function start_doris_be() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
- if ! java -version >/dev/null ||
- [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]];
then
- sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
+ if install_java && [[ -z "${JAVA_HOME}" ]]; then
+ # default to use java-8
+ JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' |
sed -n '1p')"
+ export JAVA_HOME
fi
- JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed
-n '1p')"
- export JAVA_HOME
ASAN_SYMBOLIZER_PATH="$(command -v llvm-symbolizer)"
if [[ -z "${ASAN_SYMBOLIZER_PATH}" ]]; then
ASAN_SYMBOLIZER_PATH='/var/local/ldb-toolchain/bin/llvm-symbolizer'; fi
export ASAN_SYMBOLIZER_PATH
diff --git a/regression-test/pipeline/performance/compile.sh
b/regression-test/pipeline/performance/compile.sh
index d5228b7938a..cfeae6af5f2 100644
--- a/regression-test/pipeline/performance/compile.sh
+++ b/regression-test/pipeline/performance/compile.sh
@@ -109,6 +109,10 @@ if sudo docker ps -a --no-trunc | grep "${docker_name}";
then
fi
rm -f custom_env.sh
cp
"${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/custom_env.sh
.
+if [[ "${target_branch}" == "master" ]]; then
+ echo "export JAVA_HOME=/usr/lib/jvm/jdk-17.0.2" >>custom_env.sh
+ jdk17_str="&& cd /usr/lib/jvm/ && wget --continue -t3
https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz
&& yum install -y pigz && tar -I pigz -xf openjdk-17.0.2_linux-x64_bin.tar.gz
&& cd -"
+fi
rm -rf "${teamcity_build_checkoutDir}"/output
set -x
# shellcheck disable=SC2086
@@ -131,7 +135,7 @@ sudo docker run -i --rm \
&& export CCACHE_REMOTE_STORAGE=file:///root/ccache \
&& export EXTRA_CXX_FLAGS=-O3 \
&& export USE_JEMALLOC='ON' \
- && export ENABLE_PCH=OFF \
+ && export ENABLE_PCH=OFF ${jdk17_str}\
&& export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \
&& bash build.sh --fe --be --clean 2>&1 | tee build.log"
set +x
diff --git a/regression-test/pipeline/performance/prepare.sh
b/regression-test/pipeline/performance/prepare.sh
index 0390ca6007f..0a78ed136ed 100644
--- a/regression-test/pipeline/performance/prepare.sh
+++ b/regression-test/pipeline/performance/prepare.sh
@@ -28,6 +28,7 @@ set +x
export teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
export commit_id_from_checkout="%build.vcs.number%"
export target_branch='%teamcity.pullRequest.target.branch%'
+export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64/"
if [[ -f
"${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/prepare.sh
]]; then
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
bash prepare.sh
@@ -65,11 +66,21 @@ if [[ "${commit_id_from_trigger}" !=
"${commit_id_from_checkout}" ]]; then
commit_id_from_trigger is outdate"
exit 1
fi
+
# shellcheck source=/dev/null
source "$(bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'get')"
+# shellcheck source=/dev/null
+# install_java
+source
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+
if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0;
else echo "INFO: no skip"; fi
-if [[ "${target_branch}" == "master" || "${target_branch}" == "branch-2.0" ]];
then
- echo "INFO: PR target branch ${target_branch} is in (master, branch-2.0)"
+if [[ "${target_branch}" == "master" ]]; then
+ echo "INFO: PR target branch ${target_branch}"
+ install_java
+ JAVA_HOME="${JAVA_HOME:-$(find /usr/lib/jvm -maxdepth 1 -type d -name
'java-17-*' | sed -n '1p')}"
+ bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'set' "export JAVA_HOME=\"${JAVA_HOME}\""
+elif [[ "${target_branch}" == "branch-2.0" ]]; then
+ echo "INFO: PR target branch ${target_branch}"
else
echo "WARNING: PR target branch ${target_branch} is NOT in (master,
branch-2.0), skip pipeline."
bash
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh
'set' "export skip_pipeline=true"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
