This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 19876094e feat(HDFS): Add hdfs_service related configs in server.ini
and run.sh (#1705)
19876094e is described below
commit 19876094e941b23894422ae3fdd4b102a9a0bc44
Author: Yingchun Lai <[email protected]>
AuthorDate: Thu Nov 30 17:44:11 2023 +0800
feat(HDFS): Add hdfs_service related configs in server.ini and run.sh
(#1705)
This patch adds `hdfs_service` related configs to `src/server/config.ini`,
and
add an option `--hdfs_service_args` to `start_onebox` command of `run.sh`.
---
run.sh | 10 ++++++++++
src/server/config.ini | 10 +++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/run.sh b/run.sh
index 74d44b4b4..3c86f669c 100755
--- a/run.sh
+++ b/run.sh
@@ -690,6 +690,8 @@ function usage_start_onebox()
echo "
./src/server/config.ini in production env"
echo " --use_product_config"
echo " use the product config template"
+ echo " --hdfs_service_args"
+ echo " set the 'args' value of section
'[block_service.hdfs_service]', it's a space separated HDFS namenode host:port
and path string, for example: '127.0.0.1:8020 /pegasus'. Default is empty"
echo " --opts"
echo " update configs before start onebox, the configs
are in the form of 'key1=value1,key2=value2'"
}
@@ -705,6 +707,7 @@ function run_start_onebox()
SERVER_PATH=${BUILD_LATEST_DIR}/output/bin/pegasus_server
CONFIG_FILE=""
USE_PRODUCT_CONFIG=false
+ HDFS_SERVICE_ARGS=""
OPTS=""
while [[ $# > 0 ]]; do
@@ -747,6 +750,11 @@ function run_start_onebox()
--use_product_config)
USE_PRODUCT_CONFIG=true
;;
+ --hdfs_service_args)
+ HDFS_SERVICE_ARGS="$2 $3"
+ shift
+ shift
+ ;;
--opts)
OPTS="$2"
shift
@@ -776,6 +784,7 @@ function run_start_onebox()
exit 1
fi
+ echo "HDFS_SERVICE_ARGS $HDFS_SERVICE_ARGS"
if [ $USE_PRODUCT_CONFIG == "true" ]; then
[ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini
[ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist";
exit 1; }
@@ -789,6 +798,7 @@ function run_start_onebox()
sed -i 's/%{slog.dir}//g' ${ROOT}/config-server.ini
sed -i 's/%{data.dirs}//g' ${ROOT}/config-server.ini
sed -i 's@%{home.dir}@'"$HOME"'@g' ${ROOT}/config-server.ini
+ sed -i 's@%{hdfs_service_args}@'"${HDFS_SERVICE_ARGS}"'@g'
${ROOT}/config-server.ini
for i in $(seq ${META_COUNT})
do
meta_port=$((34600+i))
diff --git a/src/server/config.ini b/src/server/config.ini
index 99ca1ea1c..b9eabf71b 100644
--- a/src/server/config.ini
+++ b/src/server/config.ini
@@ -301,9 +301,17 @@ stateful = true
;; recommand using cluster name as the root
cold_backup_root = %{cluster.name}
max_concurrent_uploading_file_count = 10
-
max_concurrent_bulk_load_downloading_count = 5
+ hdfs_read_limit_rate_mb_per_sec = 200
+ hdfs_read_batch_size_bytes = 67108864
+ hdfs_write_limit_rate_mb_per_sec = 200
+ hdfs_write_batch_size_bytes = 67108864
+
+[block_service.hdfs_service]
+ type = hdfs_service
+ args = %{hdfs_service_args}
+
[pegasus.server]
rocksdb_verbose_log = false
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
