This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 9b6db59b feat: zookeeper client support kerberos (#950)
9b6db59b is described below
commit 9b6db59bab1cf9c40d38a63575160aed50134559
Author: liguohao <[email protected]>
AuthorDate: Fri Apr 22 16:00:54 2022 +0800
feat: zookeeper client support kerberos (#950)
---
.github/workflows/ci-pull-request.yaml | 2 +-
rdsn | 2 +-
rfcs/2020-08-27-metric-api.md | 2 +-
scripts/clear_zk.sh | 2 +-
scripts/config_hdfs.sh | 28 +++++++++++++++++++++++++++-
scripts/pack_server.sh | 7 -------
scripts/pack_tools.sh | 6 ------
scripts/start_zk.sh | 31 +++++++++++++++++++++----------
scripts/stop_zk.sh | 2 +-
src/server/CMakeLists.txt | 2 +-
src/server/test/CMakeLists.txt | 2 +-
11 files changed, 55 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/ci-pull-request.yaml
b/.github/workflows/ci-pull-request.yaml
index b40e3cee..26944141 100644
--- a/.github/workflows/ci-pull-request.yaml
+++ b/.github/workflows/ci-pull-request.yaml
@@ -63,7 +63,7 @@ jobs:
needs: lint
runs-on: ubuntu-latest
container:
- image: ghcr.io/pegasus-kv/thirdparties-bin:ubuntu1804
+ image: apachepegasus/thirdparties-bin:ubuntu1804
defaults:
run:
shell: bash
diff --git a/rdsn b/rdsn
index 7f02509f..6cf41ac5 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit 7f02509f2cc6d66b6d27b22592607fbcbfc01ce7
+Subproject commit 6cf41ac5c7d293676fb6cc3657c217c6dc524c09
diff --git a/rfcs/2020-08-27-metric-api.md b/rfcs/2020-08-27-metric-api.md
index 58659f9c..f09577fc 100644
--- a/rfcs/2020-08-27-metric-api.md
+++ b/rfcs/2020-08-27-metric-api.md
@@ -25,7 +25,7 @@ This RFC proposes a new metric API in replace of the old
perf-counter API.
## Motivation
-The perf-counter API has bad naming convention to be parsed and queried over
the external monitoring system like [Prometheus](https://prometheus.io/), or
[open-falcon](https://open-falcon.org/).
+The perf-counter API has bad naming convention to be parsed and queried over
the external monitoring system like [Prometheus](https://prometheus.io/), or
[open-falcon](http://open-falcon.org/).
Here are some examples of the perf-counter it exposes:
diff --git a/scripts/clear_zk.sh b/scripts/clear_zk.sh
index aa02ef79..a31a90e1 100755
--- a/scripts/clear_zk.sh
+++ b/scripts/clear_zk.sh
@@ -27,7 +27,7 @@ fi
cd $INSTALL_DIR
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.6
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
if [ -d "$ZOOKEEPER_HOME" ]
then
diff --git a/scripts/config_hdfs.sh b/scripts/config_hdfs.sh
index 32dc80f3..da9b4b01 100755
--- a/scripts/config_hdfs.sh
+++ b/scripts/config_hdfs.sh
@@ -37,9 +37,35 @@ fi
JAVA_JVM_LIBRARY_DIR=$(dirname $(find "${JAVA_HOME}/" -name libjvm.so | head
-1))
export LD_LIBRARY_PATH=${JAVA_JVM_LIBRARY_DIR}:$LD_LIBRARY_PATH
+# download hdfs for unit test
+HDFS_ROOT=hadoop-2.8.4
+HDFS_TAR_NAME=${HDFS_ROOT}.tar.gz
+HDFS_TAR_MD5_VALUE="b30b409bb69185003b3babd1504ba224"
+if [ ! -f $HDFS_ROOT ]; then
+ echo "Downloading hadoop..."
+
download_url="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/hadoop-2.8.4.tar.gz";
+ if ! wget -T 5 -t 1 $download_url; then
+ echo "ERROR: download hadoop failed"
+ exit 1
+ fi
+ if [ `md5sum $HDFS_TAR_NAME | awk '{print$1}'` != $HDFS_TAR_MD5_VALUE ];
then
+ echo "check file $HDFS_TAR_NAME md5sum failed!"
+ exit 1
+ fi
+fi
+
+if [ ! -d $HDFS_ROOT ]; then
+ echo "Decompressing HDFS..."
+ if ! tar xf $HDFS_TAR_NAME; then
+ echo "ERROR: decompress hadoop failed"
+ exit 1
+ fi
+ rm -f $HDFS_TAR_NAME
+fi
+
# Set CLASSPATH to all the Hadoop jars needed to run Hadoop itself as well as
# the right configuration directory containing core-site.xml or hdfs-site.xml.
-PEGASUS_HADOOP_HOME=`pwd`/rdsn/thirdparty/build/Source/hadoop
+PEGASUS_HADOOP_HOME=`pwd`/${HDFS_ROOT}
# Prefer the HADOOP_HOME set in the environment, but use the pegasus's hadoop
dir otherwise.
export HADOOP_HOME="${HADOOP_HOME:-${PEGASUS_HADOOP_HOME}}"
if [ ! -d "$HADOOP_HOME/etc/hadoop" ] || [ ! -d "$HADOOP_HOME/share/hadoop" ];
then
diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
index 0631bb34..dc158528 100755
--- a/scripts/pack_server.sh
+++ b/scripts/pack_server.sh
@@ -105,13 +105,6 @@ copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.*
${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4
${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libsasl2.so.3 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libcom_err.so.3 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libgssapi_krb5.so.2 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libkrb5support.so.0 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libkrb5.so.3 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libk5crypto.so.3 ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/sasl2 ${pack}/bin
copy_file ./scripts/sendmail.sh ${pack}/bin
copy_file ./src/server/config.ini ${pack}/bin
copy_file ./src/server/config.min.ini ${pack}/bin
diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh
index 3e747050..4404f438 100755
--- a/scripts/pack_tools.sh
+++ b/scripts/pack_tools.sh
@@ -114,12 +114,6 @@ copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.*
${pack}/DSN_ROOT/lib/
copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4
${pack}/DSN_ROOT/lib/
copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0
${pack}/DSN_ROOT/lib/
copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/DSN_ROOT/lib
-copy_file ./rdsn/thirdparty/output/lib/libsasl2.so.3 ${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libcom_err.so.3 ${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libgssapi_krb5.so.2
${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libkrb5support.so.0
${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libkrb5.so.3 ${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libk5crypto.so.3 ${pack}/DSN_ROOT/lib/
copy_file `get_stdcpp_lib $custom_gcc` ${pack}/DSN_ROOT/lib/
pack_tools_lib() {
diff --git a/scripts/start_zk.sh b/scripts/start_zk.sh
index 385b3fb6..f8dcf54e 100755
--- a/scripts/start_zk.sh
+++ b/scripts/start_zk.sh
@@ -40,32 +40,43 @@ fi
cd "$INSTALL_DIR" || exit
-if [ ! -f zookeeper-3.4.6.tar.gz ]; then
+ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin
+ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz
+ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a"
+
+if [ ! -f $ZOOKEEPER_TAR_NAME ]; then
echo "Downloading zookeeper..."
-
download_url="https://github.com/XiaoMi/pegasus-common/releases/download/deps/zookeeper-3.4.6.tar.gz";
+
download_url="http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz";
if ! wget -T 5 -t 1 $download_url; then
echo "ERROR: download zookeeper failed"
exit 1
fi
+ if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` !=
$ZOOKEEPER_TAR_MD5_VALUE ]; then
+ echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!"
+ exit 1
+ fi
fi
-if [ ! -d zookeeper-3.4.6 ]; then
+if [ ! -d $ZOOKEEPER_ROOT ]; then
echo "Decompressing zookeeper..."
- if ! tar xf zookeeper-3.4.6.tar.gz; then
+ if ! tar xf $ZOOKEEPER_TAR_NAME; then
echo "ERROR: decompress zookeeper failed"
exit 1
fi
fi
-ZOOKEEPER_HOME=$(pwd)/zookeeper-3.4.6
+ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT
ZOOKEEPER_PORT=$PORT
-cp "$ZOOKEEPER_HOME"/conf/zoo_sample.cfg "$ZOOKEEPER_HOME"/conf/zoo.cfg
-sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@"
"$ZOOKEEPER_HOME"/conf/zoo.cfg
-sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@"
"$ZOOKEEPER_HOME"/conf/zoo.cfg
+cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg
+sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@"
$ZOOKEEPER_HOME/conf/zoo.cfg
+sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@"
$ZOOKEEPER_HOME/conf/zoo.cfg
+echo "admin.enableServer=false" >> $ZOOKEEPER_HOME/conf/zoo.cfg
+echo "4lw.commands.whitelist=ruok" >> $ZOOKEEPER_HOME/conf/zoo.cfg
-mkdir -p "$ZOOKEEPER_HOME"/data
-"$ZOOKEEPER_HOME"/bin/zkServer.sh start
+mkdir -p $ZOOKEEPER_HOME/data
+$ZOOKEEPER_HOME/bin/zkServer.sh start
+sleep 1
zk_check_count=0
while true; do
diff --git a/scripts/stop_zk.sh b/scripts/stop_zk.sh
index 7382615b..da06a91b 100755
--- a/scripts/stop_zk.sh
+++ b/scripts/stop_zk.sh
@@ -27,7 +27,7 @@ fi
cd $INSTALL_DIR
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.6
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
if [ -d "$ZOOKEEPER_HOME" ]
then
diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt
index 4b6c70de..d6fc99a4 100644
--- a/src/server/CMakeLists.txt
+++ b/src/server/CMakeLists.txt
@@ -41,13 +41,13 @@ set(MY_PROJ_LIBS
pegasus_reporter
pegasus_base
pegasus_client_static
- zookeeper_mt
event
galaxy-fds-sdk-cpp
PocoNet
PocoFoundation
PocoNetSSL
PocoJSON
+ hashtable
)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt
index 4bd8558b..836aee47 100644
--- a/src/server/test/CMakeLists.txt
+++ b/src/server/test/CMakeLists.txt
@@ -49,7 +49,6 @@ set(MY_PROJ_LIBS
pegasus_reporter
RocksDB::rocksdb
pegasus_client_static
- zookeeper_mt
event
galaxy-fds-sdk-cpp
PocoNet
@@ -59,6 +58,7 @@ set(MY_PROJ_LIBS
pegasus_base
gtest
gmock
+ hashtable
)
add_definitions(-DPEGASUS_UNIT_TEST)
add_definitions(-DENABLE_FAIL)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
