This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d6f2c2c6b8c [opt](ci) start dockers in parallel (#41338)
d6f2c2c6b8c is described below
commit d6f2c2c6b8caea7768185cb2476cb7ff364235b6
Author: Socrates <[email protected]>
AuthorDate: Fri Sep 27 22:49:59 2024 +0800
[opt](ci) start dockers in parallel (#41338)
## Proposed changes
Start docker in parallel to reduce external pipeline time
---
.../docker-compose/hive/scripts/hive-metastore.sh | 38 ++++-
docker/thirdparties/run-thirdparties-docker.sh | 180 +++++++++++++++++----
2 files changed, 175 insertions(+), 43 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index 5cf08f1f429..46d1437db29 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -27,6 +27,15 @@ sleep 10s
# new cases should use separate dir
hadoop fs -mkdir -p /user/doris/suites/
+lockfile1 = "mnt/scripts/run-data.lock"
+
+# wait lockfile
+while [ -f "$lockfile1" ]; do
+ sleep 10
+done
+
+touch "$lockfile1"
+
DATA_DIR="/mnt/scripts/data/"
find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {}
sh -c '
START_TIME=$(date +%s)
@@ -36,6 +45,17 @@ find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0
-n 1 -P 10 -I {} sh
echo "Script: {} executed in $EXECUTION_TIME seconds"
'
+rm -f "$lockfile1"
+
+lockfile2 = "mnt/scripts/download-data.lock"
+
+# wait lockfile
+while [ -f "$lockfile2" ]; do
+ sleep 10
+done
+
+touch "$lockfile2"
+
# if you test in your local，better use # to annotation section about tpch1.db
if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
echo "/mnt/scripts/tpch1.db does not exist"
@@ -48,11 +68,6 @@ else
echo "/mnt/scripts/tpch1.db exist, continue !"
fi
-# put data file
-## put tpch1
-hadoop fs -mkdir -p /user/doris/
-hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
-
# paimon data file is small and update frequently, so we download it every time
rm -rf "/mnt/scripts/paimon1"
echo "/mnt/scripts/paimon1 does not exist"
@@ -62,9 +77,6 @@ tar -zxf paimon1.tar.gz
rm -rf paimon1.tar.gz
cd -
-## put paimon1
-hadoop fs -put /mnt/scripts/paimon1 /user/doris/
-
# download tvf_data
if [[ ! -d "/mnt/scripts/tvf_data" ]]; then
echo "/mnt/scripts/tvf_data does not exist"
@@ -77,6 +89,16 @@ else
echo "/mnt/scripts/tvf_data exist, continue !"
fi
+rm -f "$lockfile2"
+
+# put data file
+## put tpch1
+hadoop fs -mkdir -p /user/doris/
+hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
+
+## put paimon1
+hadoop fs -put /mnt/scripts/paimon1 /user/doris/
+
## put tvf_data
hadoop fs -put /mnt/scripts/tvf_data /user/doris/
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index 36d2486a1c3..e6a5bb43771 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -182,7 +182,7 @@ for element in "${COMPONENTS_ARR[@]}"; do
fi
done
-if [[ "${RUN_ES}" -eq 1 ]]; then
+start_es() {
# elasticsearch
cp "${ROOT}"/docker-compose/elasticsearch/es.yaml.tpl
"${ROOT}"/docker-compose/elasticsearch/es.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/elasticsearch/es.yaml
@@ -205,9 +205,9 @@ if [[ "${RUN_ES}" -eq 1 ]]; then
sudo chmod -R 777 "${ROOT}"/docker-compose/elasticsearch/config
sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml
--env-file "${ROOT}"/docker-compose/elasticsearch/es.env up -d --remove-orphans
fi
-fi
+}
-if [[ "${RUN_MYSQL}" -eq 1 ]]; then
+start_mysql() {
# mysql 5.7
cp "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml.tpl
"${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
@@ -217,9 +217,9 @@ if [[ "${RUN_MYSQL}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/mysql/data/
sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
--env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env up -d
fi
-fi
+}
-if [[ "${RUN_PG}" -eq 1 ]]; then
+start_pg() {
# pg 14
cp "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml.tpl
"${ROOT}"/docker-compose/postgresql/postgresql-14.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/postgresql/postgresql-14.yaml
@@ -229,9 +229,9 @@ if [[ "${RUN_PG}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/postgresql/data/data
sudo docker compose -f
"${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file
"${ROOT}"/docker-compose/postgresql/postgresql-14.env up -d
fi
-fi
+}
-if [[ "${RUN_ORACLE}" -eq 1 ]]; then
+start_oracle() {
# oracle
cp "${ROOT}"/docker-compose/oracle/oracle-11.yaml.tpl
"${ROOT}"/docker-compose/oracle/oracle-11.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/oracle/oracle-11.yaml
@@ -241,9 +241,9 @@ if [[ "${RUN_ORACLE}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/oracle/data/
sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml
--env-file "${ROOT}"/docker-compose/oracle/oracle-11.env up -d
fi
-fi
+}
-if [[ "${RUN_DB2}" -eq 1 ]]; then
+start_db2() {
# db2
cp "${ROOT}"/docker-compose/db2/db2.yaml.tpl
"${ROOT}"/docker-compose/db2/db2.yaml
sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/db2/db2.yaml
@@ -253,9 +253,9 @@ if [[ "${RUN_DB2}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/db2/data/
sudo docker compose -f "${ROOT}"/docker-compose/db2/db2.yaml
--env-file "${ROOT}"/docker-compose/db2/db2.env up -d
fi
-fi
+}
-if [[ "${RUN_OCEANBASE}" -eq 1 ]]; then
+start_oceanbase() {
# oceanbase
cp "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml.tpl
"${ROOT}"/docker-compose/oceanbase/oceanbase.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/oceanbase/oceanbase.yaml
@@ -265,9 +265,9 @@ if [[ "${RUN_OCEANBASE}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/oceanbase/data/
sudo docker compose -f
"${ROOT}"/docker-compose/oceanbase/oceanbase.yaml --env-file
"${ROOT}"/docker-compose/oceanbase/oceanbase.env up -d
fi
-fi
+}
-if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
+start_sqlserver() {
# sqlserver
cp "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml.tpl
"${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
@@ -277,9 +277,9 @@ if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/sqlserver/data/
sudo docker compose -f
"${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file
"${ROOT}"/docker-compose/sqlserver/sqlserver.env up -d
fi
-fi
+}
-if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
+start_clickhouse() {
# clickhouse
cp "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml.tpl
"${ROOT}"/docker-compose/clickhouse/clickhouse.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/clickhouse/clickhouse.yaml
@@ -289,9 +289,9 @@ if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
sudo mkdir -p "${ROOT}"/docker-compose/clickhouse/data/
sudo docker compose -f
"${ROOT}"/docker-compose/clickhouse/clickhouse.yaml --env-file
"${ROOT}"/docker-compose/clickhouse/clickhouse.env up -d
fi
-fi
+}
-if [[ "${RUN_KAFKA}" -eq 1 ]]; then
+start_kafka() {
# kafka
KAFKA_CONTAINER_ID="${CONTAINER_UID}kafka"
eth_name=$(ifconfig -a | grep -E "^eth[0-9]" | sort -k1.4n | awk -F ':'
'{print $1}' | head -n 1)
@@ -320,9 +320,9 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
sleep 10s
create_kafka_topics "${KAFKA_CONTAINER_ID}" "${IP_HOST}"
fi
-fi
+}
-if [[ "${RUN_HIVE2}" -eq 1 ]]; then
+start_hive2() {
# hive2
# If the doris cluster you need to test is single-node, you can use the
default values; If the doris cluster you need to test is composed of multiple
nodes, then you need to set the IP_HOST according to the actual situation of
your machine
#default value
@@ -360,9 +360,9 @@ if [[ "${RUN_HIVE2}" -eq 1 ]]; then
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -p ${CONTAINER_UID}hive2 -f
"${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive-2x.env up --build --remove-orphans -d
fi
-fi
+}
-if [[ "${RUN_HIVE3}" -eq 1 ]]; then
+start_hive3() {
# hive3
# If the doris cluster you need to test is single-node, you can use the
default values; If the doris cluster you need to test is composed of multiple
nodes, then you need to set the IP_HOST according to the actual situation of
your machine
#default value
@@ -399,16 +399,16 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -p ${CONTAINER_UID}hive3 -f
"${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive-3x.env up --build --remove-orphans -d
fi
-fi
+}
-if [[ "${RUN_SPARK}" -eq 1 ]]; then
+start_spark() {
sudo docker compose -f "${ROOT}"/docker-compose/spark/spark.yaml down
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/spark/spark.yaml up
--build --remove-orphans -d
fi
-fi
+}
-if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
+start_iceberg() {
# iceberg
ICEBERG_DIR=${ROOT}/docker-compose/iceberg
cp "${ROOT}"/docker-compose/iceberg/iceberg.yaml.tpl
"${ROOT}"/docker-compose/iceberg/iceberg.yaml
@@ -433,9 +433,9 @@ if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/iceberg/iceberg.yaml
--env-file "${ROOT}"/docker-compose/iceberg/iceberg.env up -d
fi
-fi
+}
-if [[ "${RUN_HUDI}" -eq 1 ]]; then
+start_hudi() {
# hudi
cp "${ROOT}"/docker-compose/hudi/hudi.yaml.tpl
"${ROOT}"/docker-compose/hudi/hudi.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/hudi/hudi.yaml
@@ -455,9 +455,9 @@ if [[ "${RUN_HUDI}" -eq 1 ]]; then
docker exec -it adhoc-1 /bin/bash
/var/scripts/setup_demo_container_adhoc_1.sh
docker exec -it adhoc-2 /bin/bash
/var/scripts/setup_demo_container_adhoc_2.sh
fi
-fi
+}
-if [[ "${RUN_TRINO}" -eq 1 ]]; then
+start_trino() {
# trino
trino_docker="${ROOT}"/docker-compose/trino
TRINO_CONTAINER_ID="${CONTAINER_UID}trino"
@@ -530,9 +530,9 @@ if [[ "${RUN_TRINO}" -eq 1 ]]; then
# execute create table sql
docker exec -it ${TRINO_CONTAINER_ID} /bin/bash -c 'trino -f
/scripts/create_trino_table.sql'
fi
-fi
+}
-if [[ "${RUN_MARIADB}" -eq 1 ]]; then
+start_mariadb() {
# mariadb
cp "${ROOT}"/docker-compose/mariadb/mariadb-10.yaml.tpl
"${ROOT}"/docker-compose/mariadb/mariadb-10.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/mariadb/mariadb-10.yaml
@@ -542,9 +542,9 @@ if [[ "${RUN_MARIADB}" -eq 1 ]]; then
sudo rm "${ROOT}"/docker-compose/mariadb/data/* -rf
sudo docker compose -f
"${ROOT}"/docker-compose/mariadb/mariadb-10.yaml --env-file
"${ROOT}"/docker-compose/mariadb/mariadb-10.env up -d
fi
-fi
+}
-if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then
+start_lakesoul() {
echo "RUN_LAKESOUL"
cp "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml.tpl
"${ROOT}"/docker-compose/lakesoul/lakesoul.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/lakesoul/lakesoul.yaml
@@ -575,9 +575,9 @@ if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then
cd LakeSoul/rust
cargo test load_tpch_data --package lakesoul-datafusion --features=ci
-- --nocapture
fi
-fi
+}
-if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
+start_kerberos() {
echo "RUN_KERBEROS"
cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl
"${ROOT}"/docker-compose/kerberos/kerberos.yaml
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/kerberos/kerberos.yaml
@@ -599,4 +599,114 @@ if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
echo '172.31.71.26 hadoop-master-2' >> /etc/hosts
sleep 2
fi
+}
+
+echo "starting dockers in parrallel"
+
+declare -A pids
+
+if [[ "${RUN_ES}" -eq 1 ]]; then
+ start_es > start_es.log 2>&1 &
+ pids["es"]=$!
+fi
+
+if [[ "${RUN_MYSQL}" -eq 1 ]]; then
+ start_mysql > start_mysql.log 2>&1 &
+ pids["mysql"]=$!
+fi
+
+if [[ "${RUN_PG}" -eq 1 ]]; then
+ start_pg > start_pg.log 2>&1 &
+ pids["pg"]=$!
+fi
+
+if [[ "${RUN_ORACLE}" -eq 1 ]]; then
+ start_oracle > start_oracle.log 2>&1 &
+ pids["oracle"]=$!
+fi
+
+if [[ "${RUN_DB2}" -eq 1 ]]; then
+ start_db2 > start_db2.log 2>&1 &
+ pids["db2"]=$!
+fi
+
+if [[ "${RUN_OCEANBASE}" -eq 1 ]]; then
+ start_oceanbase > start_oceanbase.log 2>&1 &
+ pids["oceanbase"]=$!
+fi
+
+if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
+ start_sqlserver > start_sqlserver.log 2>&1 &
+ pids["sqlserver"]=$!
+fi
+
+if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
+ start_clickhouse > start_clickhouse.log 2>&1 &
+ pids["clickhouse"]=$!
+fi
+
+if [[ "${RUN_KAFKA}" -eq 1 ]]; then
+ start_kafka > start_kafka.log 2>&1 &
+ pids["kafka"]=$!
+fi
+
+if [[ "${RUN_HIVE2}" -eq 1 ]]; then
+ start_hive2 > start_hive2.log 2>&1 &
+ pids["hive2"]=$!
fi
+
+if [[ "${RUN_HIVE3}" -eq 1 ]]; then
+ start_hive3 > start_hive3.log 2>&1 &
+ pids["hive3"]=$!
+fi
+
+if [[ "${RUN_SPARK}" -eq 1 ]]; then
+ start_spark > start_spark.log 2>&1 &
+ pids["spark"]=$!
+fi
+
+if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
+ start_iceberg > start_icerberg.log 2>&1 &
+ pids["iceberg"]=$!
+fi
+
+if [[ "${RUN_HUDI}" -eq 1 ]]; then
+ start_hudi > start_hudi.log 2>&1 &
+ pids["hudi"]=$!
+fi
+
+if [[ "${RUN_TRINO}" -eq 1 ]]; then
+ start_trino > start_trino.log 2>&1 &
+ pids["trino"]=$!
+fi
+
+if [[ "${RUN_MARIADB}" -eq 1 ]]; then
+ start_mariadb > start_mariadb.log 2>&1 &
+ pids["mariadb"]=$!
+fi
+
+if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then
+ start_lakesoul > start_lakesoule.log 2>&1 &
+ pids["lakesoul"]=$!
+fi
+
+if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
+ start_kerberos > start_kerberos.log 2>&1 &
+ pids["kerberos"]=$!
+fi
+
+echo "waiting all dockers starting done"
+
+for compose in "${!pids[@]}"; do
+ # prevent wait return 1 make the script exit
+ status=0
+ wait "${pids[$compose]}" || status=$?
+ if [ $status -ne 0 ] && [ $compose != "db2" ]; then
+ echo "docker $compose started failed with status $status"
+ echo "print start_${compose}.log"
+ cat start_${compose}.log
+ exit 1
+ fi
+done
+
+echo "all dockers started successfully"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
