This is an automated email from the ASF dual-hosted git repository. yuchenhe pushed a commit to branch v2.1.2 in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 82062d5cdeb2f6688c55358f755b9c3f51fd53dc Author: Zhang Yifan <[email protected]> AuthorDate: Tue Dec 1 14:43:23 2020 +0800 feat(backup): support using hdfs to backup/restore pegasus tables (#647) --- scripts/pack_server.sh | 34 +++++++++++++++++++++++++ scripts/travis.sh | 2 ++ src/server/available_detector.cpp | 6 ++--- src/server/config.ini | 8 +++--- src/server/config.min.ini | 8 +++--- src/server/pegasus_manual_compact_service.cpp | 4 +-- src/server/test/config.ini | 10 +++----- src/server/test/manual_compact_service_test.cpp | 3 ++- src/shell/CMakeLists.txt | 2 ++ src/shell/command_helper.h | 2 +- 10 files changed, 57 insertions(+), 22 deletions(-) diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 0dbc08d..c78471d 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -24,6 +24,7 @@ function usage() echo " -h" echo " -p|--update-package-template <minos-package-template-file-path>" echo " -g|--custom-gcc" + echo " -k|--keytab-file" exit 0 } @@ -79,6 +80,7 @@ if [ -n "$MINOS_CONFIG_FILE" ]; then fi custom_gcc="false" +keytab_file="" while [[ $# > 0 ]]; do option_key="$1" @@ -93,6 +95,10 @@ while [[ $# > 0 ]]; do -h|--help) usage ;; + -k|--keytab-file) + keytab_file="$2" + shift + ;; esac shift done @@ -105,6 +111,7 @@ copy_file ./DSN_ROOT/lib/libdsn_utils.so ${pack}/bin copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.48 ${pack}/bin copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/bin copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin +copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/bin copy_file ./scripts/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin @@ -116,6 +123,33 @@ copy_file `get_system_lib server aio` ${pack}/bin/`get_system_libname server aio copy_file `get_system_lib server zstd` ${pack}/bin/`get_system_libname server zstd` copy_file `get_system_lib server lz4` ${pack}/bin/`get_system_libname server lz4` +# Pack hadoop-related files. +# If you want to use hdfs service to backup/restore/bulkload pegasus tables, +# you need to set env ${HADOOP_HOME}, edit ${HADOOP_HOME}/etc/hadoop/core-site.xml, +# and specify the keytab file. +if [ -n "$HADOOP_HOME" ] && [ -n "$keytab_file" ]; then + mkdir -p ${pack}/hadoop + copy_file $keytab_file ${pack}/hadoop + copy_file ${HADOOP_HOME}/etc/hadoop/core-site.xml ${pack}/hadoop + if [ -d $HADOOP_HOME/share/hadoop ]; then + for f in ${HADOOP_HOME}/share/hadoop/common/lib/*.jar; do + copy_file $f ${pack}/hadoop + done + for f in ${HADOOP_HOME}/share/hadoop/common/*.jar; do + copy_file $f ${pack}/hadoop + done + for f in ${HADOOP_HOME}/share/hadoop/hdfs/lib/*.jar; do + copy_file $f ${pack}/hadoop + done + for f in ${HADOOP_HOME}/share/hadoop/hdfs/*.jar; do + copy_file $f ${pack}/hadoop + done + fi +else + echo "Couldn't find env ${HADOOP_HOME} or no valid keytab file was specified, + hadoop-related files were not packed." +fi + DISTRIB_ID=$(cat /etc/*-release | grep DISTRIB_ID | awk -F'=' '{print $2}') DISTRIB_RELEASE=$(cat /etc/*-release | grep DISTRIB_RELEASE | awk -F'=' '{print $2}') if [ -n "$DISTRIB_ID" ] && [ -n "$DISTRIB_RELEASE" ]; then diff --git a/scripts/travis.sh b/scripts/travis.sh index 7300a43..557ce1d 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -32,6 +32,8 @@ if [ "$modified" ]; then exit 1 fi +export LD_LIBRARY_PATH=/usr/local/lib/jvm/openjdk11/lib/server:$LD_LIBRARY_PATH + "${root}"/run.sh build -c --skip_thirdparty --disable_gperf && ./run.sh test --on_travis ret=$? if [ $ret ]; then diff --git a/src/server/available_detector.cpp b/src/server/available_detector.cpp index 6a6aefb..f19bf95 100644 --- a/src/server/available_detector.cpp +++ b/src/server/available_detector.cpp @@ -19,11 +19,11 @@ #include "available_detector.h" -#include <iomanip> #include <algorithm> -#include <sstream> #include <dsn/dist/replication/duplication_common.h> -#include <dsn/utility/time_utils.h> +#include <dsn/utils/time_utils.h> +#include <iomanip> +#include <sstream> #include "base/pegasus_key_schema.h" #include "result_writer.h" diff --git a/src/server/config.ini b/src/server/config.ini index b4ec419..5dbda09 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -7,7 +7,7 @@ name = meta arguments = ports = 34601 - pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_FDS_SERVICE + pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE run = true count = 1 @@ -16,7 +16,7 @@ name = replica arguments = ports = 34801 - pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_FDS_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG + pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG run = true count = 1 @@ -130,8 +130,8 @@ worker_priority = THREAD_xPRIORITY_NORMAL worker_count = 8 -[threadpool.THREAD_POOL_FDS_SERVICE] - name = fds_service +[threadpool.THREAD_POOL_BLOCK_SERVICE] + name = block_service worker_count = 8 worker_priority = THREAD_xPRIORITY_NORMAL worker_count = 8 diff --git a/src/server/config.min.ini b/src/server/config.min.ini index 9a4d805..a12fe8d 100644 --- a/src/server/config.min.ini +++ b/src/server/config.min.ini @@ -6,13 +6,13 @@ type = meta name = meta ports = @META_PORT@ - pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_LOCAL_SERVICE + pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE [apps.replica] type = replica name = replica ports = @REPLICA_PORT@ - pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_LOCAL_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG + pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG [apps.collector] name = collector @@ -76,8 +76,8 @@ name = rep_long worker_count = 2 -[threadpool.THREAD_POOL_LOCAL_SERVICE] - name = local_service +[threadpool.THREAD_POOL_BLOCK_SERVICE] + name = block_service worker_count = 1 [threadpool.THREAD_POOL_COMPACT] diff --git a/src/server/pegasus_manual_compact_service.cpp b/src/server/pegasus_manual_compact_service.cpp index 5aa2adf..f20f0ca 100644 --- a/src/server/pegasus_manual_compact_service.cpp +++ b/src/server/pegasus_manual_compact_service.cpp @@ -19,11 +19,11 @@ #include "pegasus_manual_compact_service.h" -#include <dsn/utility/string_conv.h> -#include <dsn/utility/time_utils.h> #include <dsn/dist/fmt_logging.h> #include <dsn/dist/replication/replication.codes.h> #include <dsn/tool-api/async_calls.h> +#include <dsn/utility/string_conv.h> +#include <dsn/utils/time_utils.h> #include "base/pegasus_const.h" #include "pegasus_server_impl.h" diff --git a/src/server/test/config.ini b/src/server/test/config.ini index 4185601..aeff7e2 100644 --- a/src/server/test/config.ini +++ b/src/server/test/config.ini @@ -10,7 +10,7 @@ type = replica name = replica arguments = ports = @REPLICA_PORT@ -pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_LOCAL_SERVICE,THREAD_POOL_FDS_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_SLOG,THREAD_POOL_PLOG +pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_SLOG,THREAD_POOL_PLOG run = true count = 1 @@ -97,12 +97,8 @@ partitioned = false worker_priority = THREAD_xPRIORITY_NORMAL worker_count = 4 -[threadpool.THREAD_POOL_LOCAL_SERVICE] -name = local_service -worker_count = 1 - -[threadpool.THREAD_POOL_FDS_SERVICE] -name = fds_service +[threadpool.THREAD_POOL_BLOCK_SERVICE] +name = block_service worker_count = 1 [task..default] diff --git a/src/server/test/manual_compact_service_test.cpp b/src/server/test/manual_compact_service_test.cpp index 9cd1d24..210075c 100644 --- a/src/server/test/manual_compact_service_test.cpp +++ b/src/server/test/manual_compact_service_test.cpp @@ -17,9 +17,10 @@ * under the License. */ +#include <dsn/utils/time_utils.h> + #include "pegasus_server_test_base.h" #include "server/pegasus_manual_compact_service.h" -#include <dsn/utility/time_utils.h> namespace pegasus { namespace server { diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index d5bf429..7a7a2af 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -36,6 +36,7 @@ set(MY_PROJ_LIBS dsn_utils dsn.block_service.local dsn.block_service.fds + dsn.block_service.hdfs dsn.block_service dsn.failure_detector pegasus_client_static @@ -47,6 +48,7 @@ set(MY_PROJ_LIBS pegasus_geo_lib RocksDB::rocksdb s2 + hdfs ) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini") diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 4181685..0a71300 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -35,7 +35,7 @@ #include <dsn/dist/replication/mutation_log_tool.h> #include <dsn/perf_counter/perf_counter_utils.h> #include <dsn/utility/string_view.h> -#include <dsn/utility/time_utils.h> +#include <dsn/utils/time_utils.h> #include <rrdb/rrdb.code.definition.h> #include <rrdb/rrdb_types.h> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
