This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 8da45c6 feat(backup): support using hdfs to backup/restore pegasus
tables (#647)
8da45c6 is described below
commit 8da45c6069f59bd78c860caf47b2dec559ed9f2d
Author: Zhang Yifan <[email protected]>
AuthorDate: Tue Dec 1 14:43:23 2020 +0800
feat(backup): support using hdfs to backup/restore pegasus tables (#647)
---
rdsn | 2 +-
scripts/pack_server.sh | 34 +++++++++++++++++++++++++
scripts/travis.sh | 2 ++
src/server/available_detector.cpp | 6 ++---
src/server/config.ini | 8 +++---
src/server/config.min.ini | 8 +++---
src/server/pegasus_manual_compact_service.cpp | 4 +--
src/server/test/config.ini | 10 +++-----
src/server/test/manual_compact_service_test.cpp | 3 ++-
src/shell/CMakeLists.txt | 2 ++
src/shell/command_helper.h | 2 +-
11 files changed, 58 insertions(+), 23 deletions(-)
diff --git a/rdsn b/rdsn
index 501fe55..64b9e11 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit 501fe552f960b6afa1e1dd4b5d1bf33c87379470
+Subproject commit 64b9e11703aea7acfa771037cbc3ad8b363abfaa
diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
index 0dbc08d..c78471d 100755
--- a/scripts/pack_server.sh
+++ b/scripts/pack_server.sh
@@ -24,6 +24,7 @@ function usage()
echo " -h"
echo " -p|--update-package-template <minos-package-template-file-path>"
echo " -g|--custom-gcc"
+ echo " -k|--keytab-file"
exit 0
}
@@ -79,6 +80,7 @@ if [ -n "$MINOS_CONFIG_FILE" ]; then
fi
custom_gcc="false"
+keytab_file=""
while [[ $# > 0 ]]; do
option_key="$1"
@@ -93,6 +95,10 @@ while [[ $# > 0 ]]; do
-h|--help)
usage
;;
+ -k|--keytab-file)
+ keytab_file="$2"
+ shift
+ ;;
esac
shift
done
@@ -105,6 +111,7 @@ copy_file ./DSN_ROOT/lib/libdsn_utils.so ${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.48 ${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4
${pack}/bin
copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin
+copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/bin
copy_file ./scripts/sendmail.sh ${pack}/bin
copy_file ./src/server/config.ini ${pack}/bin
@@ -116,6 +123,33 @@ copy_file `get_system_lib server aio`
${pack}/bin/`get_system_libname server aio
copy_file `get_system_lib server zstd` ${pack}/bin/`get_system_libname server
zstd`
copy_file `get_system_lib server lz4` ${pack}/bin/`get_system_libname server
lz4`
+# Pack hadoop-related files.
+# If you want to use hdfs service to backup/restore/bulkload pegasus tables,
+# you need to set env ${HADOOP_HOME}, edit
${HADOOP_HOME}/etc/hadoop/core-site.xml,
+# and specify the keytab file.
+if [ -n "$HADOOP_HOME" ] && [ -n "$keytab_file" ]; then
+ mkdir -p ${pack}/hadoop
+ copy_file $keytab_file ${pack}/hadoop
+ copy_file ${HADOOP_HOME}/etc/hadoop/core-site.xml ${pack}/hadoop
+ if [ -d $HADOOP_HOME/share/hadoop ]; then
+ for f in ${HADOOP_HOME}/share/hadoop/common/lib/*.jar; do
+ copy_file $f ${pack}/hadoop
+ done
+ for f in ${HADOOP_HOME}/share/hadoop/common/*.jar; do
+ copy_file $f ${pack}/hadoop
+ done
+ for f in ${HADOOP_HOME}/share/hadoop/hdfs/lib/*.jar; do
+ copy_file $f ${pack}/hadoop
+ done
+ for f in ${HADOOP_HOME}/share/hadoop/hdfs/*.jar; do
+ copy_file $f ${pack}/hadoop
+ done
+ fi
+else
+ echo "Couldn't find env ${HADOOP_HOME} or no valid keytab file was
specified,
+ hadoop-related files were not packed."
+fi
+
DISTRIB_ID=$(cat /etc/*-release | grep DISTRIB_ID | awk -F'=' '{print $2}')
DISTRIB_RELEASE=$(cat /etc/*-release | grep DISTRIB_RELEASE | awk -F'='
'{print $2}')
if [ -n "$DISTRIB_ID" ] && [ -n "$DISTRIB_RELEASE" ]; then
diff --git a/scripts/travis.sh b/scripts/travis.sh
index 7300a43..557ce1d 100755
--- a/scripts/travis.sh
+++ b/scripts/travis.sh
@@ -32,6 +32,8 @@ if [ "$modified" ]; then
exit 1
fi
+export LD_LIBRARY_PATH=/usr/local/lib/jvm/openjdk11/lib/server:$LD_LIBRARY_PATH
+
"${root}"/run.sh build -c --skip_thirdparty --disable_gperf && ./run.sh test
--on_travis
ret=$?
if [ $ret ]; then
diff --git a/src/server/available_detector.cpp
b/src/server/available_detector.cpp
index 6a6aefb..f19bf95 100644
--- a/src/server/available_detector.cpp
+++ b/src/server/available_detector.cpp
@@ -19,11 +19,11 @@
#include "available_detector.h"
-#include <iomanip>
#include <algorithm>
-#include <sstream>
#include <dsn/dist/replication/duplication_common.h>
-#include <dsn/utility/time_utils.h>
+#include <dsn/utils/time_utils.h>
+#include <iomanip>
+#include <sstream>
#include "base/pegasus_key_schema.h"
#include "result_writer.h"
diff --git a/src/server/config.ini b/src/server/config.ini
index b4ec419..5dbda09 100644
--- a/src/server/config.ini
+++ b/src/server/config.ini
@@ -7,7 +7,7 @@
name = meta
arguments =
ports = 34601
- pools =
THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_FDS_SERVICE
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE
run = true
count = 1
@@ -16,7 +16,7 @@
name = replica
arguments =
ports = 34801
- pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_FDS_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
run = true
count = 1
@@ -130,8 +130,8 @@
worker_priority = THREAD_xPRIORITY_NORMAL
worker_count = 8
-[threadpool.THREAD_POOL_FDS_SERVICE]
- name = fds_service
+[threadpool.THREAD_POOL_BLOCK_SERVICE]
+ name = block_service
worker_count = 8
worker_priority = THREAD_xPRIORITY_NORMAL
worker_count = 8
diff --git a/src/server/config.min.ini b/src/server/config.min.ini
index 9a4d805..a12fe8d 100644
--- a/src/server/config.min.ini
+++ b/src/server/config.min.ini
@@ -6,13 +6,13 @@
type = meta
name = meta
ports = @META_PORT@
- pools =
THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_LOCAL_SERVICE
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE
[apps.replica]
type = replica
name = replica
ports = @REPLICA_PORT@
- pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_LOCAL_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
[apps.collector]
name = collector
@@ -76,8 +76,8 @@
name = rep_long
worker_count = 2
-[threadpool.THREAD_POOL_LOCAL_SERVICE]
- name = local_service
+[threadpool.THREAD_POOL_BLOCK_SERVICE]
+ name = block_service
worker_count = 1
[threadpool.THREAD_POOL_COMPACT]
diff --git a/src/server/pegasus_manual_compact_service.cpp
b/src/server/pegasus_manual_compact_service.cpp
index 5aa2adf..f20f0ca 100644
--- a/src/server/pegasus_manual_compact_service.cpp
+++ b/src/server/pegasus_manual_compact_service.cpp
@@ -19,11 +19,11 @@
#include "pegasus_manual_compact_service.h"
-#include <dsn/utility/string_conv.h>
-#include <dsn/utility/time_utils.h>
#include <dsn/dist/fmt_logging.h>
#include <dsn/dist/replication/replication.codes.h>
#include <dsn/tool-api/async_calls.h>
+#include <dsn/utility/string_conv.h>
+#include <dsn/utils/time_utils.h>
#include "base/pegasus_const.h"
#include "pegasus_server_impl.h"
diff --git a/src/server/test/config.ini b/src/server/test/config.ini
index ab557e7..2f8d405 100644
--- a/src/server/test/config.ini
+++ b/src/server/test/config.ini
@@ -10,7 +10,7 @@ type = replica
name = replica
arguments =
ports = @REPLICA_PORT@
-pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_LOCAL_SERVICE,THREAD_POOL_FDS_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_SLOG,THREAD_POOL_PLOG
run = true
count = 1
@@ -97,12 +97,8 @@ partitioned = false
worker_priority = THREAD_xPRIORITY_NORMAL
worker_count = 4
-[threadpool.THREAD_POOL_LOCAL_SERVICE]
-name = local_service
-worker_count = 1
-
-[threadpool.THREAD_POOL_FDS_SERVICE]
-name = fds_service
+[threadpool.THREAD_POOL_BLOCK_SERVICE]
+name = block_service
worker_count = 1
[task..default]
diff --git a/src/server/test/manual_compact_service_test.cpp
b/src/server/test/manual_compact_service_test.cpp
index 9cd1d24..210075c 100644
--- a/src/server/test/manual_compact_service_test.cpp
+++ b/src/server/test/manual_compact_service_test.cpp
@@ -17,9 +17,10 @@
* under the License.
*/
+#include <dsn/utils/time_utils.h>
+
#include "pegasus_server_test_base.h"
#include "server/pegasus_manual_compact_service.h"
-#include <dsn/utility/time_utils.h>
namespace pegasus {
namespace server {
diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt
index d5bf429..7a7a2af 100644
--- a/src/shell/CMakeLists.txt
+++ b/src/shell/CMakeLists.txt
@@ -36,6 +36,7 @@ set(MY_PROJ_LIBS
dsn_utils
dsn.block_service.local
dsn.block_service.fds
+ dsn.block_service.hdfs
dsn.block_service
dsn.failure_detector
pegasus_client_static
@@ -47,6 +48,7 @@ set(MY_PROJ_LIBS
pegasus_geo_lib
RocksDB::rocksdb
s2
+ hdfs
)
set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini")
diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h
index 93aaf41..65403e7 100644
--- a/src/shell/command_helper.h
+++ b/src/shell/command_helper.h
@@ -35,7 +35,7 @@
#include <dsn/dist/replication/mutation_log_tool.h>
#include <dsn/perf_counter/perf_counter_utils.h>
#include <dsn/utility/string_view.h>
-#include <dsn/utility/time_utils.h>
+#include <dsn/utils/time_utils.h>
#include <rrdb/rrdb.code.definition.h>
#include <rrdb/rrdb_types.h>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
