This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new dbe66a23a chore(pack): support to pack server binaries separately
(#2034)
dbe66a23a is described below
commit dbe66a23a9bd53a70da8191a08132c40fdcb6f56
Author: Samunroyu <[email protected]>
AuthorDate: Fri May 31 11:01:12 2024 +0800
chore(pack): support to pack server binaries separately (#2034)
This patch adds a new flag `--separate_servers` to indicate whether to pack
`pegasus_collector`，`pegasus_meta_server` and `pegasus_replica_server`
binaries, otherwise a combined `pegasus_server` binary will be packed in the
pegasus_server_xxx.tar.
When build server in option `--separate_servers`,the corresponding option
to use pack command is:
```
./run.sh pack_server -s or ./run.sh pack_server --separate_servers
./run.sh pack_tools -s or ./run.sh pack_tools --separate_servers
```
---
.github/actions/build_pegasus/action.yaml | 4 ++--
.github/workflows/lint_and_test_cpp.yaml | 3 ++-
scripts/pack_client.sh | 6 ------
scripts/pack_common.sh | 6 +++++-
scripts/pack_server.sh | 35 +++++++++++++++++++------------
scripts/pack_tools.sh | 24 ++++++++++++++-------
6 files changed, 48 insertions(+), 30 deletions(-)
diff --git a/.github/actions/build_pegasus/action.yaml
b/.github/actions/build_pegasus/action.yaml
index ad640f2fa..d0093ffb6 100644
--- a/.github/actions/build_pegasus/action.yaml
+++ b/.github/actions/build_pegasus/action.yaml
@@ -38,12 +38,12 @@ runs:
shell: bash
- name: Pack Server
run: |
- ./run.sh pack_server -j
+ ./run.sh pack_server -j ${PACK_OPTIONS}
rm -rf pegasus-server-*
shell: bash
- name: Pack Tools
run: |
- ./run.sh pack_tools -j
+ ./run.sh pack_tools -j ${PACK_OPTIONS}
rm -rf pegasus-tools-*
shell: bash
- name: Clear Build Files
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index dba649771..8f6e0e300 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -418,7 +418,8 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
- BUILD_OPTIONS: -t debug --test
+ BUILD_OPTIONS: -t debug --test --separate_servers
+ PACK_OPTIONS: --separate_servers
container:
image: apache/pegasus:thirdparties-bin-centos7-${{ github.base_ref }}
steps:
diff --git a/scripts/pack_client.sh b/scripts/pack_client.sh
index 66021465c..c28bb7dfd 100755
--- a/scripts/pack_client.sh
+++ b/scripts/pack_client.sh
@@ -39,12 +39,6 @@ then
exit 1
fi
-if [ ! -f ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ]
-then
- echo "ERROR: ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
not found"
- exit 1
-fi
-
if [ ! -f ${BUILD_LATEST_DIR}/CMakeCache.txt ]
then
echo "ERROR: ${BUILD_LATEST_DIR}/CMakeCache.txt not found"
diff --git a/scripts/pack_common.sh b/scripts/pack_common.sh
index d22555d33..d1080f48d 100755
--- a/scripts/pack_common.sh
+++ b/scripts/pack_common.sh
@@ -20,7 +20,11 @@ set -e
function get_stdcpp_lib()
{
- libname=`ldd ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
2>/dev/null | grep libstdc++`
+ if [[ $2 == "false" ]]; then
+ libname=`ldd
${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server 2>/dev/null | grep
libstdc++`
+ else
+ libname=`ldd
${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server
2>/dev/null | grep libstdc++`
+ fi
libname=`echo $libname | cut -f1 -d" "`
if [ $1 = "true" ]; then
gcc_path=`which gcc`
diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
index 775002128..5f1172e91 100755
--- a/scripts/pack_server.sh
+++ b/scripts/pack_server.sh
@@ -27,6 +27,7 @@ function usage() {
echo " -g|--custom-gcc"
echo " -k|--keytab-file"
echo " -j|--use-jemalloc"
+ echo " -s|--separate_servers"
exit 0
}
@@ -39,11 +40,6 @@ if [ ! -f src/include/pegasus/git_commit.h ]; then
exit 1
fi
-if [ ! -f ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ]; then
- echo "ERROR: ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
not found"
- exit 1
-fi
-
if [ ! -f ${BUILD_LATEST_DIR}/CMakeCache.txt ]; then
echo "ERROR: ${BUILD_LATEST_DIR}/CMakeCache.txt not found"
exit 1
@@ -77,7 +73,8 @@ fi
custom_gcc="false"
keytab_file=""
-use_jemalloc="off"
+use_jemalloc="false"
+separate_servers="false"
while [[ $# > 0 ]]; do
option_key="$1"
@@ -97,7 +94,10 @@ while [[ $# > 0 ]]; do
shift
;;
-j | --use-jemalloc)
- use_jemalloc="on"
+ use_jemalloc="true"
+ ;;
+ -s | --separate_servers)
+ separate_servers="true"
;;
*)
echo "ERROR: unknown option \"$option_key\""
@@ -110,12 +110,17 @@ while [[ $# > 0 ]]; do
done
mkdir -p ${pack}/bin
-copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
${pack}/bin
+if [[ $separate_servers == "false" ]]; then
+ copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
${pack}/bin
+else
+ copy_file
${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server
${pack}/bin
+ copy_file
${BUILD_LATEST_DIR}/output/bin/pegasus_replica_server/pegasus_replica_server
${pack}/bin
+fi
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_meta_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_replica_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_utils.so ${pack}/bin
-if [ "$use_jemalloc" == "on" ]; then
+if [ "$use_jemalloc" == "true" ]; then
copy_file ${THIRDPARTY_ROOT}/output/lib/libjemalloc.so.2 ${pack}/bin
copy_file ${THIRDPARTY_ROOT}/output/lib/libprofiler.so.0 ${pack}/bin
else
@@ -130,14 +135,18 @@ copy_file ./src/server/config.ini ${pack}/bin
copy_file ./src/server/config.min.ini ${pack}/bin
copy_file ./scripts/config_hdfs.sh ${pack}/bin
-copy_file "$(get_stdcpp_lib $custom_gcc)" "${pack}/bin"
+copy_file "$(get_stdcpp_lib $custom_gcc $separate_servers)" "${pack}/bin"
pack_server_lib() {
- pack_system_lib "${pack}/bin" server "$1"
+ if [[ $2 == "false" ]]; then
+ pack_system_lib "${pack}/bin" server "$1"
+ else
+ pack_system_lib "${pack}/bin" meta_server "$1"
+ fi
}
-pack_server_lib crypto
-pack_server_lib ssl
+pack_server_lib crypto $separate_servers
+pack_server_lib ssl $separate_servers
# Pack hadoop-related files.
# If you want to use hdfs service to backup/restore/bulkload pegasus tables,
diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh
index f948f1700..de1d6cdf2 100755
--- a/scripts/pack_tools.sh
+++ b/scripts/pack_tools.sh
@@ -27,6 +27,7 @@ function usage()
echo " -p|--update-package-template <minos-package-template-file-path>"
echo " -g|--custom-gcc"
echo " -j|--use-jemalloc"
+ echo " -s|--separate_servers"
exit 0
}
@@ -82,7 +83,8 @@ if [ -n "$MINOS_CONFIG_FILE" ]; then
fi
custom_gcc="false"
-use_jemalloc="off"
+use_jemalloc="false"
+separate_servers="false"
while [[ $# > 0 ]]; do
option_key="$1"
@@ -98,7 +100,10 @@ while [[ $# > 0 ]]; do
usage
;;
-j|--use-jemalloc)
- use_jemalloc="on"
+ use_jemalloc="true"
+ ;;
+ -s | --separate_servers)
+ separate_servers="true"
;;
*)
echo "ERROR: unknown option \"$option_key\""
@@ -114,7 +119,12 @@ mkdir -p ${pack}
copy_file ./run.sh ${pack}/
mkdir -p ${pack}/bin
-cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_server ${pack}/bin/
+if [[ $separate_servers == "false" ]]; then
+ copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server
${pack}/bin
+else
+ copy_file
${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server
${pack}/bin
+ copy_file
${BUILD_LATEST_DIR}/output/bin/pegasus_replica_server/pegasus_replica_server
${pack}/bin
+fi
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_shell ${pack}/bin/
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_bench ${pack}/bin/
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_kill_test ${pack}/bin/
@@ -124,7 +134,7 @@ cp -v -r
${BUILD_LATEST_DIR}/output/bin/pegasus_pressureclient ${pack}/bin/
mkdir -p ${pack}/lib
copy_file ${BUILD_LATEST_DIR}/output/lib/*.so* ${pack}/lib/
-if [ "$use_jemalloc" == "on" ]; then
+if [ "$use_jemalloc" == "true" ]; then
copy_file ${THIRDPARTY_ROOT}/output/lib/libjemalloc.so.2 ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/libprofiler.so.0 ${pack}/lib/
else
@@ -134,14 +144,14 @@ fi
copy_file ${THIRDPARTY_ROOT}/output/lib/libboost*.so.1.69.0 ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/libhdfs* ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/librocksdb.so.8 ${pack}/lib/
-copy_file `get_stdcpp_lib $custom_gcc` ${pack}/lib/
+copy_file `get_stdcpp_lib $custom_gcc $separate_servers` ${pack}/lib/
pack_tools_lib() {
pack_system_lib "${pack}/lib" shell "$1"
}
-pack_tools_lib crypto
-pack_tools_lib ssl
+pack_tools_lib crypto $separate_servers
+pack_tools_lib ssl $separate_servers
chmod -x ${pack}/lib/*
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
