This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c323bc44ff [feature](docker)add be init script option (#16909)
c323bc44ff is described below
commit c323bc44ff24402152b5c19c69502651ea10ed55
Author: FreeOnePlus <[email protected]>
AuthorDate: Tue Apr 18 20:03:18 2023 +0800
[feature](docker)add be init script option (#16909)
---
docker/runtime/be/resource/entry_point.sh | 171 ++++++++++++++++++++++++------
docker/runtime/be/resource/init_be.sh | 142 ++++++++-----------------
2 files changed, 184 insertions(+), 129 deletions(-)
diff --git a/docker/runtime/be/resource/entry_point.sh
b/docker/runtime/be/resource/entry_point.sh
index ab2ec6a3d3..2627111d50 100644
--- a/docker/runtime/be/resource/entry_point.sh
+++ b/docker/runtime/be/resource/entry_point.sh
@@ -59,22 +59,111 @@ docker_setup_env() {
fi
}
+# Check the variables required for startup
+docker_required_variables_env() {
+ declare -g RUN_TYPE
+ if [ -n "$BUILD_TYPE" ]; then
+ RUN_TYPE="K8S"
+ if [[ $BUILD_TYPE =~ ^([kK]8[sS])$ ]]; then
+ doris_warn "BUILD_TYPE" $BUILD_TYPE
+ else
+ doris_error "BUILD_TYPE rule error！example: [k8s], Default Value:
docker"
+ fi
+ export RUN_TYPE=${RUN_TYPE}
+ return
+ fi
+
+ if [[ -n "$FE_SERVERS" && -n "$BE_ADDR" ]]; then
+ RUN_TYPE="ELECTION"
+ if [[ $FE_SERVERS =~
^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$
|| $FE_SERVERS =~
^.+:([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]
[...]
+ doris_warn "FE_SERVERS" $FE_SERVERS
+ else
+ doris_error "FE_SERVERS rule error！example:
\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
+ fi
+ if [[ $BE_ADDR =~
^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$
|| $BE_ADDR =~
^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{
[...]
+ doris_warn "BE_ADDR" $BE_ADDR
+ else
+ doris_error "BE_ADDR rule error！example:
\$BE_IP:\$HEARTBEAT_SERVICE_PORT"
+ fi
+ export RUN_TYPE=${RUN_TYPE}
+ return
+ fi
+
+ if [[ -n "$FE_MASTER_IP" && -n "$BE_IP" && -n "$BE_PORT" ]]; then
+ RUN_TYPE="ASSIGN"
+ if [[ $FE_MASTER_IP =~
^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ ||
$FE_MASTER_IP =~
^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|
[...]
+ doris_warn "FE_MASTER_IP" $FE_MASTER_IP
+ else
+ doris_error "FE_MASTER_IP rule error！example: \$FE_MASTER_IP"
+ fi
+ if [[ $BE_IP =~
^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $BE_IP =~
^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$
]]; then
+ doris_warn "BE_IP" $BE_IP
+ else
+ doris_error "BE_IP rule error！example: \$BE_IP"
+ fi
+ if [[ $BE_PORT =~ ^[1-6]{0,1}[0-9]{1,4}$ ]]; then
+ doris_warn "BE_PORT" $BE_PORT
+ else
+ doris_error "BE_PORT rule error！example: \$BE_PORT."
+ fi
+ export RUN_TYPE=${RUN_TYPE}
+ return
+ fi
+
+
+ doris_error EOF "
+ Note that you did not configure the required parameters!
+ plan 1:
+ BUILD_TYPE
+ plan 2:
+ FE_SERVERS & BE_ADDR
+ plan 3:
+ FE_MASTER_IP & FE_MASTER_PORT & BE_IP & BE_PORT"
+ EOF
+}
+
get_doris_args() {
- local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print
$0}'))
- for i in "${feServerArray[@]}"; do
+ declare -g MASTER_FE_IP CURRENT_BE_IP CURRENT_BE_PORT PRIORITY_NETWORKS
+ if [ $RUN_TYPE == "ELECTION" ]; then
+ local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," ");
print $0}'))
+ for i in "${feServerArray[@]}"; do
val=${i}
val=${val// /}
- tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, "");
print$1}')
+ tmpFeName=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, "");
print$1}')
tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
- feIpArray[$tmpFeId]=${tmpFeIp}
- done
+ tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, "");
print$3}')
+ check_arg "TMP_FE_NAME" $tmpFeName
+ feIpArray[$tmpFeName]=${tmpFeIp}
+ done
+
+ FE_MASTER_IP=${feIpArray[1]}
+ check_arg "FE_MASTER_IP" $FE_MASTER_IP
+ BE_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
+ check_arg "BE_IP" $BE_IP
+ BE_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
+ check_arg "BE_PORT" $BE_PORT
+
+ elif [ $RUN_TYPE == "ASSIGN" ]; then
+ check_arg "FE_MASTER_IP" $FE_MASTER_IP
+ check_arg "BE_IP" $BE_IP
+ check_arg "BE_PORT" $BE_PORT
+ fi
+
+ PRIORITY_NETWORKS=$(echo "${BE_IP}" | awk -F '.'
'{print$1"."$2"."$3".0/24"}')
+ check_arg "PRIORITY_NETWORKS" $PRIORITY_NETWORKS
+
+ # export be args
+ export MASTER_FE_IP=${FE_MASTER_IP}
+ export CURRENT_BE_IP=${BE_IP}
+ export CURRENT_BE_PORT=${BE_PORT}
+ export PRIORITY_NETWORKS=${PRIORITY_NETWORKS}
- declare -g MASTER_FE_IP BE_HOST_IP BE_HEARTBEAT_PORT
- MASTER_FE_IP=${feIpArray[1]}
- doris_note "masterFe = ${MASTER_FE_IP}"
- BE_HOST_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
- BE_HEARTBEAT_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, "");
print$2}')
- doris_note "be_addr = ${BE_HOST_IP}:${BE_HEARTBEAT_PORT}"
+ doris_note "MASTER_FE_IP ${MASTER_FE_IP}"
+ doris_note "CURRENT_BE_IP ${CURRENT_BE_IP}"
+ doris_note "CURRENT_BE_PORT ${CURRENT_BE_PORT}"
+ doris_note "PRIORITY_NETWORKS ${PRIORITY_NETWORKS}"
+
+ check_be_status true
}
# Execute sql script, passed via stdin
@@ -83,32 +172,36 @@ get_doris_args() {
# ie: docker_process_sql --database=mydb <my-file.sql
docker_process_sql() {
set +e
- mysql -uroot -P9030 -h${MASTER_FE_IP} --comments "$@" 2>/dev/null
+ mysql -uroot -P9030 -h${MASTER_FE_IP} --comments "$@" 2>&1
}
check_be_status() {
- set +e
- local is_fe_start=false
- for i in {1..300}; do
- if [[ $(($i % 20)) == 1 ]]; then
- doris_warn "start check be status~"
- fi
- docker_process_sql <<<"show backends;" | grep
"[[:space:]]${BE_HOST_IP}[[:space:]]" | grep
"[[:space:]]${BE_HEARTBEAT_PORT}[[:space:]]" | grep "[[:space:]]true[[:space:]]"
- be_join_status=$?
- if [[ "${be_join_status}" == 0 ]]; then
- doris_note "Verify that BE is registered to FE successfully"
- is_fe_start=true
- break
- else
- if [[ $(($i % 20)) == 1 ]]; then
- doris_note "register is failed, wait next~"
- fi
- fi
- sleep 1
- done
- if ! [[ $is_fe_start ]]; then
- doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start
Failed！"
+ set +e
+ for i in {1..300}; do
+ if [[ $1 == true ]]; then
+ docker_process_sql <<<"show frontends" | grep
"[[:space:]]${MASTER_FE_IP}[[:space:]]"
+ else
+ docker_process_sql <<<"show backends" | grep
"[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep
"[[:space:]]${CURRENT_BE_PORT}[[:space:]]" | grep "[[:space:]]true[[:space:]]"
+ fi
+ be_join_status=$?
+ if [[ "${be_join_status}" == 0 ]]; then
+ if [[ $1 == true ]]; then
+ doris_note "MASTER FE is started!"
+ else
+ doris_note "EntryPoint Check - Verify that BE is registered to FE
successfully"
+ BE_ALREADY_EXISTS=true
+ fi
+ break
fi
+ if [[ $(( $i % 20 )) == 1 ]]; then
+ if [[ $1 == true ]]; then
+ doris_note "MASTER FE is not started. retry."
+ else
+ doris_note "BE is not register. retry."
+ fi
+ fi
+ sleep 1
+ done
}
# usage: docker_process_init_files [file [file [...]]]
@@ -158,10 +251,19 @@ docker_process_init_files() {
done
}
+# Check whether the passed parameters are empty to avoid subsequent task
execution failures. At the same time,
+# enumeration checks can be added, such as checking whether a certain
parameter appears repeatedly, etc.
+check_arg() {
+ if [ -z $2 ]; then
+ doris_error "$1 is null!"
+ fi
+}
+
_main() {
- docker_setup_env
+ docker_required_variables_env
# get init args
get_doris_args
+ docker_setup_env
# Start Doris BE
{
set +e
@@ -171,6 +273,7 @@ _main() {
check_be_status
if [ -z ${DATABASE_ALREADY_EXISTS} ]; then
# run script
+ sleep 15
docker_process_init_files /docker-entrypoint-initdb.d/*
fi
diff --git a/docker/runtime/be/resource/init_be.sh
b/docker/runtime/be/resource/init_be.sh
index ee1d8f9538..fee50b2fe9 100644
--- a/docker/runtime/be/resource/init_be.sh
+++ b/docker/runtime/be/resource/init_be.sh
@@ -55,65 +55,27 @@ _is_sourced() {
}
docker_setup_env() {
+ sysctl -w vm.max_map_count=2000000
declare -g DATABASE_ALREADY_EXISTS
if [ -d "${DORIS_HOME}/be/storage/data" ]; then
DATABASE_ALREADY_EXISTS='true'
fi
}
-# Check the variables required for startup
-docker_required_variables_env() {
- if [[ $FE_SERVERS =~
^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$
]]; then
- doris_warn "FE_SERVERS" $FE_SERVERS
- else
- doris_error "FE_SERVERS rule error！example:
\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
- fi
- if [[ $BE_ADDR =~
^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$
]]; then
- doris_warn "BE_ADDR" $BE_ADDR
- else
- doris_error "BE_ADDR rule error！example:
\$BE_HOST_IP:\$HEARTBEAT_SERVICE_PORT"
- fi
-}
-
-get_doris_be_args() {
- local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print
$0}'))
- for i in "${feServerArray[@]}"; do
- val=${i}
- val=${val// /}
- tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, "");
print$1}')
- tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
- tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}')
- check_arg "tmpFeIp" $tmpFeIp
- feIpArray[$tmpFeId]=${tmpFeIp}
- check_arg "tmpFeEditLogPort" $tmpFeEditLogPort
- feEditLogPortArray[$tmpFeId]=${tmpFeEditLogPort}
- done
-
- declare -g MASTER_FE_IP BE_HOST_IP BE_HEARTBEAT_PORT PRIORITY_NETWORKS
- MASTER_FE_IP=${feIpArray[1]}
- check_arg "MASTER_FE_IP" $MASTER_FE_IP
- BE_HOST_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
- check_arg "BE_HOST_IP" $BE_HOST_IP
- BE_HEARTBEAT_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, "");
print$2}')
- check_arg "BE_HEARTBEAT_PORT" $BE_HEARTBEAT_PORT
-
- PRIORITY_NETWORKS=$(echo "${BE_HOST_IP}" | awk -F '.'
'{print$1"."$2"."$3".0/24"}')
- check_arg "priority_networks" $PRIORITY_NETWORKS
-
- doris_note "feIpArray = ${feIpArray[*]}"
- doris_note "feEditLogPortArray = ${feEditLogPortArray[*]}"
- doris_note "masterFe = ${feIpArray[1]}:${feEditLogPortArray[1]}"
- doris_note "be_addr = ${BE_HOST_IP}:${BE_HEARTBEAT_PORT}"
- doris_note "priority_networks = ${PRIORITY_NETWORKS}"
- # wait fe start
- check_be_status true
-}
-
add_priority_networks() {
doris_note "add priority_networks ${1} to ${DORIS_HOME}/be/conf/be.conf"
echo "priority_networks = ${1}" >>${DORIS_HOME}/be/conf/be.conf
}
+show_be_args(){
+ doris_note "============= init args ================"
+ doris_note "MASTER_FE_IP " ${MASTER_FE_IP}
+ doris_note "CURRENT_BE_IP " ${CURRENT_BE_IP}
+ doris_note "CURRENT_BE_PORT " ${CURRENT_BE_PORT}
+ doris_note "RUN_TYPE " ${RUN_TYPE}
+ doris_note "PRIORITY_NETWORKS " ${PRIORITY_NETWORKS}
+}
+
# Execute sql script, passed via stdin
# usage: docker_process_sql sql_script
docker_process_sql() {
@@ -135,7 +97,7 @@ register_be_to_fe() {
# check fe status
local is_fe_start=false
for i in {1..300}; do
- docker_process_sql <<<"alter system add backend
'${BE_HOST_IP}:${BE_HEARTBEAT_PORT}'"
+ docker_process_sql <<<"alter system add backend
'${CURRENT_BE_IP}:${CURRENT_BE_PORT}'"
register_be_status=$?
if [[ $register_be_status == 0 ]]; then
doris_note "BE successfully registered to FE！"
@@ -162,58 +124,48 @@ register_be_to_fe() {
fi
}
-# Check whether the passed parameters are empty to avoid subsequent task
execution failures. At the same time,
-# enumeration checks can be added, such as checking whether a certain
parameter appears repeatedly, etc.
-check_arg() {
- if [ -z $2 ]; then
- doris_error "$1 is null!"
- fi
-}
-
check_be_status() {
- set +e
- for i in {1..300}; do
- if [[ $1 == true ]]; then
- docker_process_sql <<<"show frontends" | grep
"[[:space:]]${MASTER_FE_IP}[[:space:]]"
- else
- docker_process_sql <<<"show backends" | grep
"[[:space:]]${BE_HOST_IP}[[:space:]]" | grep
"[[:space:]]${BE_HEARTBEAT_PORT}[[:space:]]"
- fi
- be_join_status=$?
- if [[ "${be_join_status}" == 0 ]]; then
- if [[ $1 == true ]]; then
- doris_note "MASTER FE is started!"
- else
- doris_note "Init Check - Verify that BE is registered to FE
successfully"
- BE_ALREADY_EXISTS=true
- fi
- break
- fi
- if [[ $(( $i % 20 )) == 1 ]]; then
- if [[ $1 == true ]]; then
- doris_note "MASTER FE is not started. retry."
- else
- doris_note "BE is not register. retry."
- fi
+ set +e
+ local is_fe_start=false
+ for i in {1..300}; do
+ if [[ $(($i % 20)) == 1 ]]; then
+ doris_warn "start check be register status~"
+ fi
+ docker_process_sql <<<"show backends;" | grep
"[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep
"[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
+ be_join_status=$?
+ if [[ "${be_join_status}" == 0 ]]; then
+ doris_note "Verify that BE is registered to FE successfully"
+ is_fe_start=true
+ return
+ else
+ if [[ $(($i % 20)) == 1 ]]; then
+ doris_note "register is failed, wait next~"
+ fi
+ fi
+ sleep 1
+ done
+ if [[ ! $is_fe_start ]]; then
+ doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start
Failed！"
fi
- sleep 1
- done
}
-_main() {
- docker_setup_env
- docker_required_variables_env
- get_doris_be_args
- if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
- add_priority_networks $PRIORITY_NETWORKS
- node_role_conf
+_main() {
+ if [[ $RUN_TYPE == "K8S" ]]; then
+ start_be.sh
+ else
+ docker_setup_env
+ if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
+ add_priority_networks $PRIORITY_NETWORKS
+ node_role_conf
+ fi
+ show_be_args
+ register_be_to_fe
+ check_be_status
+ doris_note "Ready to start BE！"
+ start_be.sh
+ exec "$@"
fi
-
- register_be_to_fe
- check_be_status
- doris_note "Ready to start BE！"
- start_be.sh
- exec "$@"
}
if ! _is_sourced; then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
