This is an automated email from the ASF dual-hosted git repository. awong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit e7cd1f6cfab4d8448ad4ebd834727c83f23f4899 Author: Andrew Wong <[email protected]> AuthorDate: Wed Jul 28 15:54:29 2021 -0700 [docker] KUDU-3307: allow using multiple directories Previously, the Docker entrypoint would by default use a single root directory, suffixed with either 'master' or 'tserver'. This makes it a poor fit for production systems that may want to use Docker, since most reasonable deployments use multple data directories, and don't colocate WALs and data. This patch adds a couple of environment variables to the entrypoint, FS_WAL_DIR and FS_DATA_DIRS, that map directly to the --fs_wal_dir and --fs_data_dirs gflags that Kudu masters and tservers know and love. To maintain compatibility with existing guides, the existing single-root DATA_DIR remains, but the new environment variables, if set, take precedence. Change-Id: I20e5f75c1f1f8280dca60de75255a4a948e44be1 Reviewed-on: http://gerrit.cloudera.org:8080/17738 Tested-by: Kudu Jenkins Reviewed-by: Bankim Bhavsar <[email protected]> --- docker/kudu-entrypoint.sh | 49 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/docker/kudu-entrypoint.sh b/docker/kudu-entrypoint.sh index 814bf0d..21485db 100755 --- a/docker/kudu-entrypoint.sh +++ b/docker/kudu-entrypoint.sh @@ -39,8 +39,18 @@ function print_help { echo " Defines the kudu-master and kudu-tserver configured master addresses." echo " Defaults to localhost." echo "DATA_DIR:" - echo " Defines the root data directory to use." + echo " Defines the root directory to use. Subdirectories are added depending on whether a " + echo " Kudu master or a Kudu tablet server is being deployed. Ignored if the FS_WAL_DIR " + echo " environment variable is set." + echo " NOTE: this variable is deprecated. FS_WAL_DIR should be used instead." echo " Defaults to /var/lib/kudu." + echo "FS_WAL_DIR:" + echo " Defines the WAL directory to use. Takes precedence over the DATA_DIR environment " + echo " variable." + echo "FS_DATA_DIRS:" + echo " Defines the data directories to use. If set, the FS_WAL_DIR environment variable " + echo " must also be set." + echo " Defaults to the value of the FS_WAL_DIR environment variable." echo "MASTER_ARGS:" echo " Defines the arguments passed to kudu-master." echo " Defaults to the value of the DEFAULT_ARGS environment variable." @@ -51,12 +61,29 @@ function print_help { echo " Defines a recommended base set of arguments." } +if [[ -z "$FS_WAL_DIR" && -n "$FS_DATA_DIRS" ]]; then + echo "If FS_DATA_DIRS is set, FS_WAL_DIR must also be set" + echo "FS_WAL_DIR: $FS_WAL_DIR" + echo "FS_DATA_DIRS: $FS_DATA_DIRS" + exit 1 +fi + +DATA_DIR=${DATA_DIR:="/var/lib/kudu"} +if [[ -n "$FS_WAL_DIR" ]]; then + # Use the WAL directory for data if a data directory is not specified. + WAL_DIR="$FS_WAL_DIR" + DATA_DIRS=${FS_DATA_DIRS:="$FS_WAL_DIR"} +else + # If no WAL directory is specified, use a subdirectory in the root directory. + WAL_DIR="$DATA_DIR/$1" + DATA_DIRS="$DATA_DIR/$1" +fi + # Define the defaults environment variables. KUDU_MASTERS=${KUDU_MASTERS:=""} -DATA_DIR=${DATA_DIR:="/var/lib/kudu"} -SERVICE_DIR="$DATA_DIR/$1" # TODO: Remove use_hybrid_clock=false when ntpd is setup. -DEFAULT_ARGS="--fs_wal_dir=$SERVICE_DIR \ +DEFAULT_ARGS="--fs_wal_dir=$WAL_DIR \ + --fs_data_dirs=$DATA_DIRS \ --webserver_doc_root=/opt/kudu/www \ --stderrthreshold=0 \ --use_hybrid_clock=false" @@ -85,6 +112,16 @@ function wait_for_master_hosts() { unset IFS } +function make_directories() { + IFS="," + mkdir -p $WAL_DIR + for DIR in $DATA_DIRS + do + mkdir -p $DIR + done + unset IFS +} + # If no arguments are passed, print the help. if [[ -z "$1" ]]; then print_help @@ -94,7 +131,7 @@ fi # Note: we use "master" and "tserver" here so the kudu-master and kudu-tserver # binaries can be manually invoked if needed. if [[ "$1" == "master" ]]; then - mkdir -p "$SERVICE_DIR" + make_directories wait_for_master_hosts # Supply --master_addresses even if a single master address is specified. if [[ -n "$KUDU_MASTERS" ]]; then @@ -102,7 +139,7 @@ if [[ "$1" == "master" ]]; then fi exec kudu master run ${DEFAULT_ARGS} ${MASTER_ARGS} elif [[ "$1" == "tserver" ]]; then - mkdir -p "$SERVICE_DIR" + make_directories wait_for_master_hosts if [[ -n "$KUDU_MASTERS" ]]; then TSERVER_ARGS="--tserver_master_addrs=$KUDU_MASTERS $TSERVER_ARGS"
