This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 41ff58d83 refactor(FDS): Terminate the FDS supporting (#1695)
41ff58d83 is described below
commit 41ff58d831e8cbf3f455995c309151ab01f9ec95
Author: Yingchun Lai <[email protected]>
AuthorDate: Wed Nov 29 20:53:43 2023 +0800
refactor(FDS): Terminate the FDS supporting (#1695)
https://github.com/apache/incubator-pegasus/issues/1644
As it known, the [FDS](https://docs.api.xiaomi.com/fds/introduction.html)
is a Xiaomi internal storage system, the community users are not possible
to use FDS as a
[Backup](https://pegasus.apache.org/zh/administration/cold-backup)
target, and since Pegasus
[2.2.0](https://github.com/apache/incubator-pegasus/releases/tag/v2.2.0)
(released in Jun, 2021), the Backup start to support using Apache HDFS as
the target storage.
This patch terminates the supporting of FDS in the next version. Of course,
we will still fix critical bugs if there are in released versions.
TODO: backup/restore modules have to update.
---
.licenserc.yaml | 1 -
cmake_modules/BaseFunctions.cmake | 3 -
scripts/pack_server.sh | 1 -
scripts/pack_tools.sh | 1 -
src/block_service/CMakeLists.txt | 1 -
src/block_service/block_service_manager.cpp | 5 -
src/block_service/fds/CMakeLists.txt | 35 -
src/block_service/fds/fds_service.cpp | 720 ------------------
src/block_service/fds/fds_service.h | 156 ----
src/block_service/test/CMakeLists.txt | 6 -
src/block_service/test/config-test.ini | 5 -
src/block_service/test/fds_service_test.cpp | 819 ---------------------
src/failure_detector/CMakeLists.txt | 4 +-
src/failure_detector/test/config-test.ini | 6 -
.../test/config-whitelist-test-failed.ini | 6 -
.../test/config-whitelist-test.ini | 6 -
src/meta/CMakeLists.txt | 6 -
src/meta/test/CMakeLists.txt | 6 -
src/meta/test/config-test.ini | 4 -
src/replica/CMakeLists.txt | 6 -
src/replica/backup/test/CMakeLists.txt | 1 -
src/replica/test/CMakeLists.txt | 1 -
src/replica/test/replica_http_service_test.cpp | 2 +-
src/server/CMakeLists.txt | 9 -
src/server/test/CMakeLists.txt | 6 -
src/server/test/config.ini | 4 -
src/shell/CMakeLists.txt | 9 -
thirdparty/CMakeLists.txt | 48 --
thirdparty/fix_fds_for_macos.patch | 14 -
29 files changed, 2 insertions(+), 1889 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 3bb82270b..185ee7757 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -65,7 +65,6 @@ header:
- 'src/replica/duplication/test/log.1.0.handle_real_private_log2'
- 'src/replica/duplication/test/log.1.0.all_loaded_are_write_empties'
# Used for patches for thirdparties.
- - 'thirdparty/fix_fds_for_macos.patch'
- 'thirdparty/fix_jemalloc_for_m1_on_macos.patch'
- 'thirdparty/fix_libevent_for_macos.patch'
- 'thirdparty/fix_prometheus-cpp_limits.patch'
diff --git a/cmake_modules/BaseFunctions.cmake
b/cmake_modules/BaseFunctions.cmake
index b55bd0576..1882353c8 100644
--- a/cmake_modules/BaseFunctions.cmake
+++ b/cmake_modules/BaseFunctions.cmake
@@ -222,9 +222,6 @@ function(dsn_setup_compiler_flags)
# use frame pointers to allow simple stack frame walking for backtraces.
# This has a small perf hit but worth it for the ability to profile in
production
add_compile_options( -fno-omit-frame-pointer)
- # -Wno-implicit-float-conversion
- # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion
- add_compile_options(-Wno-implicit-float-conversion)
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
index 200fbbacb..d39f9ac0f 100755
--- a/scripts/pack_server.sh
+++ b/scripts/pack_server.sh
@@ -114,7 +114,6 @@ copy_file
${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ${pack}/b
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_meta_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_replica_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_utils.so ${pack}/bin
-copy_file ./thirdparty/output/lib/libPoco*.so.* ${pack}/bin
if [ "$use_jemalloc" == "on" ]; then
copy_file ./thirdparty/output/lib/libjemalloc.so.2 ${pack}/bin
diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh
index 793322435..c3cc9e5d2 100755
--- a/scripts/pack_tools.sh
+++ b/scripts/pack_tools.sh
@@ -123,7 +123,6 @@ cp -v -r
${BUILD_LATEST_DIR}/output/bin/pegasus_pressureclient ${pack}/bin/
mkdir -p ${pack}/lib
copy_file ${BUILD_LATEST_DIR}/output/lib/*.so* ${pack}/lib/
-copy_file ./thirdparty/output/lib/libPoco*.so.* ${pack}/lib/
if [ "$use_jemalloc" == "on" ]; then
copy_file ./thirdparty/output/lib/libjemalloc.so.2 ${pack}/lib/
diff --git a/src/block_service/CMakeLists.txt b/src/block_service/CMakeLists.txt
index 1bcbba9ac..0ce357ce1 100644
--- a/src/block_service/CMakeLists.txt
+++ b/src/block_service/CMakeLists.txt
@@ -37,7 +37,6 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem)
dsn_add_static_library()
-add_subdirectory(fds)
add_subdirectory(hdfs)
add_subdirectory(local)
add_subdirectory(test)
diff --git a/src/block_service/block_service_manager.cpp
b/src/block_service/block_service_manager.cpp
index 07e1f8c86..a6c48509d 100644
--- a/src/block_service/block_service_manager.cpp
+++ b/src/block_service/block_service_manager.cpp
@@ -22,7 +22,6 @@
#include <vector>
#include "block_service/block_service.h"
-#include "block_service/fds/fds_service.h"
#include "block_service/hdfs/hdfs_service.h"
#include "block_service/local/local_service.h"
#include "runtime/task/task_code.h"
@@ -39,10 +38,6 @@ namespace block_service {
block_service_registry::block_service_registry()
{
- CHECK(utils::factory_store<block_filesystem>::register_factory(
- "fds_service", block_filesystem::create<fds_service>,
PROVIDER_TYPE_MAIN),
- "register fds_service failed");
-
CHECK(utils::factory_store<block_filesystem>::register_factory(
"hdfs_service", block_filesystem::create<hdfs_service>,
PROVIDER_TYPE_MAIN),
"register hdfs_service failed");
diff --git a/src/block_service/fds/CMakeLists.txt
b/src/block_service/fds/CMakeLists.txt
deleted file mode 100644
index 6058605d1..000000000
--- a/src/block_service/fds/CMakeLists.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(MY_PROJ_NAME dsn.block_service.fds)
-
-#Source files under CURRENT project directory will be automatically included.
-#You can manually set MY_PROJ_SRC to include source files under other
directories.
-set(MY_PROJ_SRC "")
-
-#Search mode for source files under CURRENT project directory ?
-#"GLOB_RECURSE" for recursive search
-#"GLOB" for non - recursive search
-set(MY_SRC_SEARCH_MODE "GLOB")
-
-set(MY_PROJ_LIBS "")
-
-#Extra files that will be installed
-set(MY_BINPLACES "")
-
-dsn_add_static_library()
-target_include_directories(dsn.block_service.fds PRIVATE
${THIRDPARTY_INSTALL_DIR}/include/fds)
diff --git a/src/block_service/fds/fds_service.cpp
b/src/block_service/fds/fds_service.cpp
deleted file mode 100644
index 8be5709ab..000000000
--- a/src/block_service/fds/fds_service.cpp
+++ /dev/null
@@ -1,720 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "block_service/fds/fds_service.h"
-
-#include <Poco/Net/HTTPResponse.h>
-#include <boost/algorithm/string/predicate.hpp>
-#include <errno.h>
-#include <fds_client_configuration.h>
-#include <galaxy_fds_client.h>
-#include <galaxy_fds_client_exception.h>
-#include <model/delete_multi_objects_result.h>
-#include <model/fds_object.h>
-#include <model/fds_object_listing.h>
-#include <model/fds_object_metadata.h>
-#include <model/fds_object_summary.h>
-#include <algorithm>
-#include <fstream>
-#include <map>
-#include <memory>
-#include <utility>
-
-#include "Poco/Exception.h"
-#include "runtime/task/async_calls.h"
-#include "utils/TokenBucket.h"
-#include "utils/autoref_ptr.h"
-#include "utils/blob.h"
-#include "utils/env.h"
-#include "utils/error_code.h"
-#include "utils/filesystem.h"
-#include "utils/flags.h"
-#include "utils/fmt_logging.h"
-#include "utils/safe_strerror_posix.h"
-#include "utils/string_conv.h"
-
-namespace dsn {
-namespace dist {
-namespace block_service {
-
-DSN_DEFINE_uint32(replication, fds_write_limit_rate, 100, "write rate limit of
fds(MB/s)");
-DSN_TAG_VARIABLE(fds_write_limit_rate, FT_MUTABLE);
-
-DSN_DEFINE_uint32(replication, fds_write_burst_size, 500, "write burst size of
fds(MB)");
-DSN_TAG_VARIABLE(fds_write_burst_size, FT_MUTABLE);
-
-DSN_DEFINE_uint32(replication, fds_read_limit_rate, 100, "read rate limit of
fds(MB/s)");
-DSN_TAG_VARIABLE(fds_read_limit_rate, FT_MUTABLE);
-
-DSN_DEFINE_uint32(replication, fds_read_batch_size, 100, "read batch size of
fds(MB)");
-DSN_TAG_VARIABLE(fds_read_batch_size, FT_MUTABLE);
-
-class utils
-{
-public:
- /*
- * read data from inputstream 'istr' then write to outputstream 'ostr'
piece by piece until
- * reach EOF, the size of each piece is specified by 'piece_size'
- */
- static size_t copy_stream(std::istream &istr, std::ostream &ostr, size_t
piece_size);
- /*
- * normalize a absolute path to a valid fds object path:
- * 1. the output_path shouldn't start with /
- * 2. the path shoudn't contain "." or ".." or "//"
- * 3. if the path is a dir, the path should ends with "/"
- * 4. if the path is root("/"), then return an empty string
- */
- static std::string path_to_fds(const std::string &input, bool is_dir);
- /*
- * normalize a fds object path to a absolute path:
- * 1. the output_path starts with /
- * 2. all the postfix / are removed if the path marks a dir
- */
- static std::string path_from_fds(const std::string &input, bool is_dir);
-};
-
-/*static*/
-size_t utils::copy_stream(std::istream &is, std::ostream &os, size_t
piece_size)
-{
- std::unique_ptr<char[]> buffer(new char[piece_size]);
- size_t length = 0;
- is.read(buffer.get(), piece_size);
- size_t got_length = is.gcount();
- while (got_length > 0) {
- length += got_length;
- os.write(buffer.get(), got_length);
- if (is && os) {
- is.read(buffer.get(), piece_size);
- got_length = is.gcount();
- } else
- got_length = 0;
- }
- return length;
-}
-
-/*static*/
-std::string utils::path_to_fds(const std::string &input, bool is_dir)
-{
- // TODO: handle the "." and ".." and "//"
- if (input.size() < 1 || input == "/")
- return std::string();
- std::string result;
- if (input.front() == '/')
- result = input.substr(1);
- else
- result = input;
-
- if (is_dir)
- result.push_back('/');
- return result;
-}
-
-/*static*/
-std::string utils::path_from_fds(const std::string &input, bool /*is_dir*/)
-{
- std::string result = input;
- if (!input.empty() && input.back() == '/')
- result.pop_back();
- return result;
-}
-
-DEFINE_TASK_CODE(LPC_FDS_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE)
-
-const std::string fds_service::FILE_LENGTH_CUSTOM_KEY =
"x-xiaomi-meta-content-length";
-const std::string fds_service::FILE_MD5_KEY = "content-md5";
-
-fds_service::fds_service()
-{
- _write_token_bucket.reset(new folly::DynamicTokenBucket());
- _read_token_bucket.reset(new folly::DynamicTokenBucket());
-}
-
-fds_service::~fds_service() {}
-
-/**
- * @brief fds_service::initialize
- * @param args: {httpServer, accessKey, secretKey, bucket}
- * @return
- */
-error_code fds_service::initialize(const std::vector<std::string> &args)
-{
- galaxy::fds::FDSClientConfiguration config;
- config.enableHttps(true);
- config.setEndpoint(args[0]);
- const std::string &access_key = args[1];
- const std::string &secret_key = args[2];
-
- _client.reset(new galaxy::fds::GalaxyFDSClient(access_key, secret_key,
config));
- _bucket_name = args[3];
- return dsn::ERR_OK;
-}
-
-#define FDS_EXCEPTION_HANDLE(ERR_REFERENCE, OPERATION, INPUT_PARAMETER)
\
- catch (const Poco::TimeoutException &ex)
\
- {
\
- LOG_ERROR("fds {} timeout: parameter({}), code({}), msg({})",
\
- OPERATION,
\
- INPUT_PARAMETER,
\
- ex.code(),
\
- ex.message());
\
- ERR_REFERENCE = ERR_TIMEOUT;
\
- }
\
- catch (const Poco::Exception &ex)
\
- {
\
- LOG_ERROR("fds {} get poco exception: parameter({}), code({}),
msg({}), what({})", \
- OPERATION,
\
- INPUT_PARAMETER,
\
- ex.code(),
\
- ex.message(),
\
- ex.what());
\
- ERR_REFERENCE = ERR_FS_INTERNAL;
\
- }
\
- catch (...)
\
- {
\
- LOG_ERROR("fds {} get unknown exception: parameter({})", OPERATION,
INPUT_PARAMETER); \
- ERR_REFERENCE = ERR_FS_INTERNAL;
\
- }
-
-dsn::task_ptr fds_service::list_dir(const ls_request &req,
- dsn::task_code code,
- const ls_callback &callback,
- dsn::task_tracker *tracker = nullptr)
-{
- ls_future_ptr t(new ls_future(code, callback, 0));
- t->set_tracker(tracker);
-
- auto list_dir_in_background = [this, req, t]() {
- ls_response resp;
- std::string fds_path = utils::path_to_fds(req.dir_name, true);
- try {
- std::shared_ptr<galaxy::fds::FDSObjectListing> result =
- _client->listObjects(_bucket_name, fds_path);
-
- while (true) {
- const std::vector<galaxy::fds::FDSObjectSummary> &objs =
result->objectSummaries();
- const std::vector<std::string> &common_prefix =
result->commonPrefixes();
- resp.err = dsn::ERR_OK;
-
- // fds listing's objects are with full-path, we must extract
the postfix to emulate
- // the filesystem structure
- for (const galaxy::fds::FDSObjectSummary &obj : objs) {
- CHECK(fds_path.empty() ||
boost::starts_with(obj.objectName(), fds_path),
- "invalid path({}) in parent({})",
- obj.objectName(),
- fds_path);
- resp.entries->push_back(
-
{utils::path_from_fds(obj.objectName().substr(fds_path.size()), false),
- false});
- }
- for (const std::string &s : common_prefix) {
- CHECK(fds_path.empty() || boost::starts_with(s, fds_path),
- "invalid path({}) in parent({})",
- s,
- fds_path);
- resp.entries->push_back(
- {utils::path_from_fds(s.substr(fds_path.size()),
true), true});
- }
-
- // list result may be paged
- if (result->truncated()) {
- auto res_temp = _client->listNextBatchOfObjects(*result);
- result.swap(res_temp);
- } else {
- break;
- }
- }
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds listObjects failed: parameter({}), code({}),
msg({})",
- req.dir_name,
- ex.code(),
- ex.what());
- resp.err = ERR_FS_INTERNAL;
- }
- FDS_EXCEPTION_HANDLE(resp.err, "listObject", req.dir_name.c_str())
-
- if (resp.err == dsn::ERR_OK && resp.entries->empty()) {
- try {
- if (_client->doesObjectExist(_bucket_name,
- utils::path_to_fds(req.dir_name,
false))) {
- LOG_ERROR("fds list_dir failed: path not dir,
parameter({})", req.dir_name);
- resp.err = ERR_INVALID_PARAMETERS;
- } else {
- LOG_ERROR("fds list_dir failed: path not found,
parameter({})", req.dir_name);
- resp.err = ERR_OBJECT_NOT_FOUND;
- }
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds doesObjectExist failed: parameter({}),
code({}), msg({})",
- req.dir_name,
- ex.code(),
- ex.what());
- resp.err = ERR_FS_INTERNAL;
- }
- FDS_EXCEPTION_HANDLE(resp.err, "doesObjectExist",
req.dir_name.c_str())
- }
-
- t->enqueue_with(resp);
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, list_dir_in_background);
- return t;
-}
-
-dsn::task_ptr fds_service::create_file(const create_file_request &req,
- dsn::task_code code,
- const create_file_callback &cb,
- dsn::task_tracker *tracker = nullptr)
-{
- create_file_future_ptr t(new create_file_future(code, cb, 0));
- t->set_tracker(tracker);
- if (req.ignore_metadata) {
- create_file_response resp;
- resp.err = dsn::ERR_OK;
- resp.file_handle =
- new fds_file_object(this, req.file_name,
utils::path_to_fds(req.file_name, false));
- t->enqueue_with(resp);
- return t;
- }
-
- auto create_file_in_background = [this, req, t]() {
- create_file_response resp;
- resp.err = ERR_IO_PENDING;
- std::string fds_path = utils::path_to_fds(req.file_name, false);
-
- dsn::ref_ptr<fds_file_object> f = new fds_file_object(this,
req.file_name, fds_path);
- resp.err = f->get_file_meta();
- if (resp.err == ERR_OK || resp.err == ERR_OBJECT_NOT_FOUND) {
- resp.err = ERR_OK;
- resp.file_handle = f;
- }
-
- t->enqueue_with(resp);
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, create_file_in_background);
- return t;
-}
-
-dsn::task_ptr fds_service::remove_path(const remove_path_request &req,
- dsn::task_code code,
- const remove_path_callback &cb,
- dsn::task_tracker *tracker)
-{
- remove_path_future_ptr callback(new remove_path_future(code, cb, 0));
- callback->set_tracker(tracker);
- auto remove_path_background = [this, req, callback]() {
- remove_path_response resp;
- resp.err = ERR_OK;
- std::string fds_path = utils::path_to_fds(req.path, true);
- bool should_remove_path = false;
-
- try {
- std::shared_ptr<galaxy::fds::FDSObjectListing> result =
- _client->listObjects(_bucket_name, fds_path);
- while (result->objectSummaries().size() <= 0 &&
result->commonPrefixes().size() <= 0 &&
- result->truncated()) {
- result = _client->listNextBatchOfObjects(*result);
- }
- const std::vector<galaxy::fds::FDSObjectSummary> &objs =
result->objectSummaries();
- const std::vector<std::string> &common_prefix =
result->commonPrefixes();
-
- if (!objs.empty() || !common_prefix.empty()) {
- // path is non-empty directory
- if (req.recursive) {
- should_remove_path = true;
- } else {
- LOG_ERROR("fds remove_path failed: dir not empty,
parameter({})", req.path);
- resp.err = ERR_DIR_NOT_EMPTY;
- }
- } else {
- if (_client->doesObjectExist(_bucket_name,
utils::path_to_fds(req.path, false))) {
- should_remove_path = true;
- } else {
- LOG_ERROR("fds remove_path failed: path not found,
parameter({})", req.path);
- resp.err = ERR_OBJECT_NOT_FOUND;
- }
- }
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds remove_path failed: parameter({}), code({}),
msg({})",
- req.path,
- ex.code(),
- ex.what());
- resp.err = ERR_FS_INTERNAL;
- }
- FDS_EXCEPTION_HANDLE(resp.err, "remove_path", req.path.c_str());
-
- if (resp.err == ERR_OK && should_remove_path) {
- fds_path = utils::path_to_fds(req.path, false);
- try {
- auto deleting = _client->deleteObjects(_bucket_name, fds_path,
false);
- if (deleting->countFailedObjects() <= 0) {
- resp.err = ERR_OK;
- } else {
- LOG_ERROR("fds remove_path failed: countFailedObjects =
{}, parameter({})",
- deleting->countFailedObjects(),
- req.path);
- resp.err = ERR_FS_INTERNAL;
- }
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds remove_path failed: parameter({}), code({}),
msg({})",
- req.path,
- ex.code(),
- ex.what());
- resp.err = ERR_FS_INTERNAL;
- }
- FDS_EXCEPTION_HANDLE(resp.err, "remove_path", req.path.c_str());
- }
-
- callback->enqueue_with(resp);
- return;
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, remove_path_background);
- return callback;
-}
-
-fds_file_object::fds_file_object(fds_service *s,
- const std::string &name,
- const std::string &fds_path)
- : block_file(name),
- _service(s),
- _fds_path(fds_path),
- _md5sum(""),
- _size(0),
- _has_meta_synced(false)
-{
-}
-
-fds_file_object::~fds_file_object() {}
-
-error_code fds_file_object::get_file_meta()
-{
- error_code err = ERR_OK;
- galaxy::fds::GalaxyFDSClient *c = _service->get_client();
- try {
- auto meta = c->getObjectMetadata(_service->get_bucket_name(),
_fds_path)->metadata();
-
- // get file length
- auto iter = meta.find(fds_service::FILE_LENGTH_CUSTOM_KEY);
- CHECK(iter != meta.end(),
- "can't find {} in object({})'s metadata",
- fds_service::FILE_LENGTH_CUSTOM_KEY,
- _fds_path);
- bool valid = dsn::buf2uint64(iter->second, _size);
- CHECK(valid, "error to get file size");
-
- // get md5 key
- iter = meta.find(fds_service::FILE_MD5_KEY);
- CHECK(iter != meta.end(),
- "can't find {} in object({})'s metadata",
- fds_service::FILE_MD5_KEY,
- _fds_path);
- _md5sum = iter->second;
-
- _has_meta_synced = true;
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- if (ex.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) {
- err = ERR_OBJECT_NOT_FOUND;
- } else {
- LOG_ERROR("fds getObjectMetadata failed: parameter({}), code({}),
msg({})",
- _name.c_str(),
- ex.code(),
- ex.what());
- err = ERR_FS_INTERNAL;
- }
- }
- FDS_EXCEPTION_HANDLE(err, "getObjectMetadata", _fds_path.c_str());
- return err;
-}
-
-error_code fds_file_object::get_content_in_batches(uint64_t start,
- int64_t length,
- /*out*/ std::ostream &os,
- /*out*/ uint64_t
&transfered_bytes)
-{
- error_code err = ERR_OK;
- transfered_bytes = 0;
-
- // get file meta if it is not synced
- if (!_has_meta_synced) {
- err = get_file_meta();
- if (ERR_OK != err) {
- return err;
- }
- }
-
- // if length = -1, it means we should transfer the whole file
- uint64_t to_transfer_bytes = (length == -1 ? _size : length);
-
- uint64_t pos = start;
- uint64_t once_transfered_bytes = 0;
- while (pos < start + to_transfer_bytes) {
- const uint64_t BATCH_SIZE = FLAGS_fds_read_batch_size << 20;
- uint64_t batch_size = std::min(BATCH_SIZE, start + to_transfer_bytes -
pos);
-
- // burst size should not be less than consume size
- const uint64_t rate = FLAGS_fds_read_limit_rate << 20;
- _service->_read_token_bucket->consumeWithBorrowAndWait(
- batch_size, rate, std::max(2 * rate, batch_size));
-
- err = get_content(pos, batch_size, os, once_transfered_bytes);
- transfered_bytes += once_transfered_bytes;
- if (err != ERR_OK || once_transfered_bytes < batch_size) {
- return err;
- }
- pos += batch_size;
- }
-
- return ERR_OK;
-}
-
-error_code fds_file_object::get_content(uint64_t pos,
- uint64_t length,
- /*out*/ std::ostream &os,
- /*out*/ uint64_t &transfered_bytes)
-{
- error_code err = ERR_OK;
- transfered_bytes = 0;
- while (true) {
- // if we have download enough or we have reach the end
- if (transfered_bytes >= length || transfered_bytes + pos >= _size) {
- return ERR_OK;
- }
-
- try {
- galaxy::fds::GalaxyFDSClient *c = _service->get_client();
- std::shared_ptr<galaxy::fds::FDSObject> obj;
- obj = c->getObject(_service->get_bucket_name(),
- _fds_path,
- pos + transfered_bytes,
- length - transfered_bytes);
- LOG_DEBUG("get object from fds succeed, remote_file({})",
_fds_path);
- std::istream &is = obj->objectContent();
- transfered_bytes += utils::copy_stream(is, os, PIECE_SIZE);
- err = ERR_OK;
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds getObject error: remote_file({}), code({}),
msg({})",
- file_name(),
- ex.code(),
- ex.what());
- if (ex.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) {
- _has_meta_synced = true;
- _md5sum = "";
- _size = 0;
- err = ERR_OBJECT_NOT_FOUND;
- } else {
- err = ERR_FS_INTERNAL;
- }
- }
- FDS_EXCEPTION_HANDLE(err, "getObject", file_name().c_str())
-
- if (err != ERR_OK) {
- return err;
- }
- }
-}
-
-error_code fds_file_object::put_content(/*in-out*/ std::istream &is,
- int64_t to_transfer_bytes,
- uint64_t &transfered_bytes)
-{
- error_code err = ERR_OK;
- transfered_bytes = 0;
- galaxy::fds::GalaxyFDSClient *c = _service->get_client();
-
- // get tokens from token bucket
- if
(!_service->_write_token_bucket->consumeWithBorrowAndWait(to_transfer_bytes,
-
FLAGS_fds_write_limit_rate << 20,
-
FLAGS_fds_write_burst_size
- << 20)) {
- LOG_INFO("the transfer count({}B) is greater than burst size({}MB), so
it is rejected by "
- "token bucket",
- to_transfer_bytes,
- FLAGS_fds_write_burst_size);
- return ERR_BUSY;
- }
-
- try {
- c->putObject(_service->get_bucket_name(), _fds_path, is,
galaxy::fds::FDSObjectMetadata());
- } catch (const galaxy::fds::GalaxyFDSClientException &ex) {
- LOG_ERROR("fds putObject error: remote_file({}), code({}), msg({})",
- file_name(),
- ex.code(),
- ex.what());
- err = ERR_FS_INTERNAL;
- }
- FDS_EXCEPTION_HANDLE(err, "putObject", file_name().c_str())
-
- if (err != ERR_OK) {
- return err;
- }
-
- LOG_INFO("start to synchronize meta data after successfully wrote data to
fds");
- err = get_file_meta();
- if (err == ERR_OK) {
- transfered_bytes = _size;
- }
- return err;
-}
-
-dsn::task_ptr fds_file_object::write(const write_request &req,
- dsn::task_code code,
- const write_callback &cb,
- dsn::task_tracker *tracker = nullptr)
-{
- write_future_ptr t(new write_future(code, cb, 0));
- t->set_tracker(tracker);
-
- add_ref();
- auto write_in_background = [this, req, t]() {
- write_response resp;
- std::istringstream is;
- is.str(std::string(req.buffer.data(), req.buffer.length()));
- resp.err = put_content(is, req.buffer.length(), resp.written_size);
-
- t->enqueue_with(resp);
- release_ref();
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, write_in_background);
- return t;
-}
-
-// TODO: handle the localfile path
-dsn::task_ptr fds_file_object::upload(const upload_request &req,
- dsn::task_code code,
- const upload_callback &cb,
- dsn::task_tracker *tracker = nullptr)
-{
- upload_future_ptr t(new upload_future(code, cb, 0));
- t->set_tracker(tracker);
-
- add_ref();
- auto upload_background = [this, req, t]() {
- const std::string &local_file = req.input_local_name;
- // get file size
- int64_t file_sz = 0;
- dsn::utils::filesystem::file_size(
- local_file, dsn::utils::FileDataType::kSensitive, file_sz);
-
- upload_response resp;
- // TODO: we can cache the whole file in buffer, then upload the buffer
rather than the
- // ifstream, because if ifstream read file beyond 60s, fds-server will
reset the session,
- // then upload will fail with error broken-pipe
- std::ifstream is(local_file, std::ios::binary | std::ios::in);
-
- if (!is.is_open()) {
- LOG_ERROR("fds upload failed: open local file({}) failed when
upload to({}), error({})",
- local_file,
- file_name(),
- ::dsn::utils::safe_strerror(errno));
- resp.err = dsn::ERR_FILE_OPERATION_FAILED;
- } else {
- resp.err = put_content(is, file_sz, resp.uploaded_size);
- is.close();
- }
-
- t->enqueue_with(resp);
- release_ref();
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, upload_background);
- return t;
-}
-
-dsn::task_ptr fds_file_object::read(const read_request &req,
- dsn::task_code code,
- const read_callback &cb,
- dsn::task_tracker *tracker = nullptr)
-{
- read_future_ptr t(new read_future(code, cb, 0));
- t->set_tracker(tracker);
-
- add_ref();
- auto read_in_background = [this, req, t]() {
- read_response resp;
- std::ostringstream os;
- uint64_t transferd_size;
- resp.err = get_content_in_batches(req.remote_pos, req.remote_length,
os, transferd_size);
- if (os.tellp() > 0) {
- std::string *output = new std::string();
- *output = os.str();
- std::shared_ptr<char> ptr((char *)output->c_str(), [output](char
*) { delete output; });
- resp.buffer.assign(std::move(ptr), 0, output->length());
- }
- t->enqueue_with(resp);
- release_ref();
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, read_in_background);
- return t;
-}
-
-// TODO: handle the localfile path
-dsn::task_ptr fds_file_object::download(const download_request &req,
- dsn::task_code code,
- const download_callback &cb,
- dsn::task_tracker *tracker = nullptr)
-{
- download_future_ptr t(new download_future(code, cb, 0));
- t->set_tracker(tracker);
- download_response resp;
-
- // TODO(yingchun): use rocksdb API to implement this.
- std::shared_ptr<std::ofstream> handle(new std::ofstream(
- req.output_local_name, std::ios::binary | std::ios::out |
std::ios::trunc));
- if (!handle->is_open()) {
- LOG_ERROR("fds download failed: fail to open localfile({}) when
download({}), error({})",
- req.output_local_name,
- _fds_path,
- ::dsn::utils::safe_strerror(errno));
- resp.err = ERR_FILE_OPERATION_FAILED;
- resp.downloaded_size = 0;
- t->enqueue_with(resp);
- return t;
- }
-
- add_ref();
- auto download_background = [this, req, handle, t]() {
- download_response resp;
- uint64_t transfered_size;
- resp.err =
- get_content_in_batches(req.remote_pos, req.remote_length, *handle,
transfered_size);
- resp.downloaded_size = 0;
- if (resp.err == ERR_OK && handle->tellp() != -1) {
- resp.downloaded_size = handle->tellp();
- }
- handle->close();
- if (resp.err != ERR_OK &&
dsn::utils::filesystem::file_exists(req.output_local_name)) {
- LOG_ERROR("fail to download file {} from fds, remove localfile {}",
- _fds_path,
- req.output_local_name);
- dsn::utils::filesystem::remove_path(req.output_local_name);
- } else if ((resp.err =
dsn::utils::filesystem::md5sum(req.output_local_name,
- resp.file_md5))
!= ERR_OK) {
- LOG_ERROR("download failed when calculate the md5sum of local file
{}",
- req.output_local_name);
- }
- t->enqueue_with(resp);
- release_ref();
- };
-
- dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, download_background);
- return t;
-}
-} // namespace block_service
-} // namespace dist
-} // namespace dsn
diff --git a/src/block_service/fds/fds_service.h
b/src/block_service/fds/fds_service.h
deleted file mode 100644
index 916cd57ef..000000000
--- a/src/block_service/fds/fds_service.h
+++ /dev/null
@@ -1,156 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef FDS_SERVICE_H
-#define FDS_SERVICE_H
-
-#include <stddef.h>
-#include <stdint.h>
-#include <chrono>
-#include <iosfwd>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "block_service/block_service.h"
-#include "runtime/task/task.h"
-#include "runtime/task/task_code.h"
-#include "utils/error_code.h"
-#include "utils/TokenBucket.h"
-
-namespace dsn {
-class task_tracker;
-} // namespace dsn
-
-namespace folly {
-using DynamicTokenBucket =
folly::BasicDynamicTokenBucket<std::chrono::steady_clock>;
-} // namespace folly
-
-namespace galaxy {
-namespace fds {
-class GalaxyFDSClient;
-}
-} // namespace galaxy
-
-namespace dsn {
-namespace dist {
-namespace block_service {
-
-class fds_service : public block_filesystem
-{
-public:
- static const std::string FILE_MD5_KEY;
- static const std::string FILE_LENGTH_KEY;
- static const std::string FILE_LENGTH_CUSTOM_KEY;
-
-public:
- fds_service();
- galaxy::fds::GalaxyFDSClient *get_client() { return _client.get(); }
- const std::string &get_bucket_name() { return _bucket_name; }
-
- virtual ~fds_service() override;
- virtual error_code initialize(const std::vector<std::string> &args)
override;
- virtual dsn::task_ptr list_dir(const ls_request &req,
- dsn::task_code code,
- const ls_callback &callback,
- dsn::task_tracker *tracker) override;
-
- virtual dsn::task_ptr create_file(const create_file_request &req,
- dsn::task_code code,
- const create_file_callback &cb,
- dsn::task_tracker *tracker) override;
-
- //
- // Attention：
- // -- remove the path directly on fds, will not enter trash
- // -- when req.path is a directory, this operation may consume much time
if there are many
- // files under this directory
- //
- virtual dsn::task_ptr remove_path(const remove_path_request &req,
- dsn::task_code code,
- const remove_path_callback &cb,
- dsn::task_tracker *tracker) override;
-
-private:
- std::shared_ptr<galaxy::fds::GalaxyFDSClient> _client;
- std::string _bucket_name;
- std::unique_ptr<folly::DynamicTokenBucket> _write_token_bucket;
- std::unique_ptr<folly::DynamicTokenBucket> _read_token_bucket;
-
- friend class fds_file_object;
-};
-
-class fds_file_object : public block_file
-{
-public:
- fds_file_object(fds_service *s, const std::string &name, const std::string
&fds_path);
-
- virtual ~fds_file_object();
- virtual uint64_t get_size() override { return _size; }
- virtual const std::string &get_md5sum() override { return _md5sum; }
-
- virtual dsn::task_ptr write(const write_request &req,
- dsn::task_code code,
- const write_callback &cb,
- dsn::task_tracker *tracker) override;
-
- virtual dsn::task_ptr read(const read_request &req,
- dsn::task_code code,
- const read_callback &cb,
- dsn::task_tracker *tracker) override;
-
- virtual dsn::task_ptr upload(const upload_request &req,
- dsn::task_code code,
- const upload_callback &cb,
- dsn::task_tracker *tracker) override;
-
- virtual dsn::task_ptr download(const download_request &req,
- dsn::task_code code,
- const download_callback &cb,
- dsn::task_tracker *tracker) override;
-
- // Possible errors:
- // - ERR_FS_INTERNAL
- // - ERR_OBJECT_NOT_FOUND
- // - ERR_TIMEOUT
- error_code get_file_meta();
-
-private:
- error_code get_content_in_batches(uint64_t start,
- int64_t length,
- /*out*/ std::ostream &os,
- /*out*/ uint64_t &transfered_bytes);
- error_code get_content(uint64_t pos,
- uint64_t length,
- /*out*/ std::ostream &os,
- /*out*/ uint64_t &transfered_bytes);
- error_code put_content(/*in-out*/ std::istream &is,
- /*int*/ int64_t to_transfer_bytes,
- /*out*/ uint64_t &transfered_bytes);
-
- fds_service *_service;
- std::string _fds_path;
- std::string _md5sum;
- uint64_t _size;
- bool _has_meta_synced;
-
- static const size_t PIECE_SIZE = 16384; // 16k
-};
-} // namespace block_service
-} // namespace dist
-} // namespace dsn
-#endif // FDS_SERVICE_H
diff --git a/src/block_service/test/CMakeLists.txt
b/src/block_service/test/CMakeLists.txt
index 78202667d..6fbfa9ca1 100644
--- a/src/block_service/test/CMakeLists.txt
+++ b/src/block_service/test/CMakeLists.txt
@@ -25,15 +25,9 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn_runtime
dsn_utils
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
gtest
gtest_main
hdfs
diff --git a/src/block_service/test/config-test.ini
b/src/block_service/test/config-test.ini
index 2acb86e2b..19a33003c 100644
--- a/src/block_service/test/config-test.ini
+++ b/src/block_service/test/config-test.ini
@@ -44,11 +44,6 @@ fast_flush = true
short_header = false
stderr_start_level = LOG_LEVEL_WARNING
-[fds_concurrent_test]
-total_files = 64
-min_size = 100
-max_size = 150
-
[threadpool.THREAD_POOL_BLOCK_SERVICE]
worker_count = 8
diff --git a/src/block_service/test/fds_service_test.cpp
b/src/block_service/test/fds_service_test.cpp
deleted file mode 100644
index 3c8d15b2b..000000000
--- a/src/block_service/test/fds_service_test.cpp
+++ /dev/null
@@ -1,819 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "block_service/fds/fds_service.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <algorithm>
-#include <fstream> // IWYU pragma: keep
-#include <iostream>
-#include <memory>
-
-#include "block_service/block_service.h"
-#include "gtest/gtest.h"
-#include "utils/autoref_ptr.h"
-#include "utils/blob.h"
-#include "utils/enum_helper.h"
-#include "utils/filesystem.h"
-#include "utils/flags.h"
-#include "utils/fmt_logging.h"
-#include "utils/process_utils.h"
-#include "utils/rand.h"
-#include "utils/safe_strerror_posix.h"
-#include "utils/strings.h"
-#include "utils/threadpool_code.h"
-#include "utils/utils.h"
-
-using namespace dsn;
-using namespace dsn::dist::block_service;
-
-DSN_DEFINE_uint64(fds_concurrent_test, min_size, 64, "");
-DSN_DEFINE_uint64(fds_concurrent_test, max_size, 64, "");
-
-static std::string example_server_address = "<server-address>";
-// please modify the the paras below to enable fds_service_test, default
fds_service_test will be
-// skipped and return true
-static std::string server_address = "<server-address>";
-static std::string access_key = "<access-key>";
-static std::string access_secret = "<access-secret>";
-static std::string bucket_name = "<test-bucket-name>";
-
-static void file_eq_compare(const std::string &fname1, const std::string
&fname2)
-{
- static const int length = 4096;
- std::shared_ptr<char> buffer(dsn::utils::make_shared_array<char>(length *
2));
- char *buf1 = buffer.get(), *buf2 = buffer.get() + length;
-
- std::ifstream ifile1(fname1.c_str(), std::ios::in | std::ios::binary);
- std::ifstream ifile2(fname2.c_str(), std::ios::in | std::ios::binary);
-
- auto file_length = [](std::ifstream &is) {
- is.seekg(0, is.end);
- int result = is.tellg();
- is.seekg(0, is.beg);
- return result;
- };
-
- int l = file_length(ifile1);
- ASSERT_EQ(l, file_length(ifile2));
-
- for (int i = 0; i < l; i += length) {
- int up_to_bytes = length < (l - i) ? length : (l - i);
- ifile1.read(buf1, up_to_bytes);
- ifile2.read(buf2, up_to_bytes);
- ASSERT_TRUE(dsn::utils::mequals(buf1, buf2, up_to_bytes));
- }
-}
-
-class FDSClientTest : public testing::Test
-{
-protected:
- virtual void SetUp() override;
- virtual void TearDown() override;
-
- struct file_desc
- {
- std::string filename;
- std::string md5;
- size_t length;
- };
-
- file_desc f1;
- file_desc f2;
- std::string local_file_for_download;
-};
-
-void FDSClientTest::SetUp()
-{
- f1.filename = "test_file";
- f2.filename = "test_2";
- local_file_for_download = "local_download";
-
- // generate a test file
- {
- int lines = rand::next_u32(1000, 2000);
- FILE *fp = fopen(f1.filename.c_str(), "wb");
- for (int i = 0; i < lines; ++i) {
- fprintf(fp, "%04d_this_is_a_simple_test_file\n", i);
- }
- fclose(fp);
-
- std::stringstream ss;
- CHECK_EQ(utils::pipe_execute((std::string("md5sum ") +
f1.filename).c_str(), ss), 0);
- ss >> f1.md5;
- // well, the string of each line in _test_file is 32
- f1.length = 32 * lines;
- }
-
- // generate another test file
- {
- int lines = rand::next_u32(10, 20);
- FILE *fp = fopen(f2.filename.c_str(), "wb");
- for (int i = 0; i < lines; ++i) {
- fprintf(fp, "%04d_this_is_a_simple_test_file\n", i);
- }
- fclose(fp);
-
- std::stringstream ss;
- CHECK_EQ(utils::pipe_execute((std::string("md5sum ") +
f2.filename).c_str(), ss), 0);
- ss >> f2.md5;
- // well, the string of each line in _test_file is 32
- f2.length = 32 * lines;
- }
-}
-
-void FDSClientTest::TearDown() {}
-
-DEFINE_TASK_CODE(lpc_btest, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT)
-
-// TODO(yingchun): add encryption test when FDSClient supports encryption.
-TEST_F(FDSClientTest, test_basic_operation)
-{
- const char *files[] = {"/fdstest/fdstest1/test1/test1",
- "/fdstest/fdstest1/test1/test2",
- "/fdstest/fdstest1/test2/test1",
- "/fdstest/fdstest1/test2/test2",
- "/fdstest/fdstest2/test2",
- "/fdstest/fdstest3",
- "/fdstest/fds_rootfile",
- nullptr};
- // ensure prefix_path is the prefix of some file in files
- std::string prefix_path = std::string("/fdstest/fdstest1/test1");
- int total_files;
-
- std::shared_ptr<fds_service> s = std::make_shared<fds_service>();
- // server, access-key, access-secret, bucket_name
- std::vector<std::string> args = {server_address, access_key,
access_secret, bucket_name};
-
- if (server_address == example_server_address) {
- // user don't specify the server-address, we just return true
- return;
- }
-
- s->initialize(args);
-
- create_file_response cf_resp;
- ls_response l_resp;
- upload_response u_resp;
- download_response d_resp;
- read_response r_resp;
- write_response w_resp;
- remove_path_response rem_resp;
-
- auto entry_cmp = [](const ls_entry &entry1, const ls_entry &entry2) {
- return entry1.entry_name < entry2.entry_name;
- };
- auto entry_vec_eq = [](const std::vector<ls_entry> &entry_vec1,
- const std::vector<ls_entry> &entry_vec2) {
- ASSERT_EQ(entry_vec1.size(), entry_vec2.size());
- for (unsigned int i = 0; i < entry_vec1.size(); ++i) {
- ASSERT_EQ(entry_vec1[i].entry_name, entry_vec2[i].entry_name);
- ASSERT_EQ(entry_vec1[i].is_directory, entry_vec2[i].is_directory)
- << "on index " << i << ", name " << entry_vec1[i].entry_name;
- }
- };
-
- // first clean all
- {
- std::cout << "clean all old files" << std::endl;
- for (int i = 0; files[i]; ++i) {
- std::cout << "delete file " << files[i] << std::endl;
- s->remove_path(remove_path_request{std::string(files[i]), true},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_TRUE(dsn::ERR_OK == rem_resp.err ||
dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err);
- }
- }
-
- // first upload all these files
- {
- std::cout << "Test upload files" << std::endl;
- for (total_files = 0; files[total_files]; ++total_files) {
- std::cout << "create and upload: " << files[total_files] <<
std::endl;
-
s->create_file(create_file_request{std::string(files[total_files]), true},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp
= r; },
- nullptr)
- ->wait();
- ASSERT_EQ(cf_resp.err, dsn::ERR_OK);
-
- cf_resp.file_handle
- ->upload(upload_request{FDSClientTest::f1.filename},
- lpc_btest,
- [&u_resp](const upload_response &r) { u_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, u_resp.err);
- ASSERT_EQ(FDSClientTest::f1.length,
cf_resp.file_handle->get_size());
- ASSERT_EQ(FDSClientTest::f1.md5,
cf_resp.file_handle->get_md5sum());
- }
-
- // create a non-exist file for read
- {
- std::cout << "create a non-exist file for read: fds_hellworld" <<
std::endl;
- s->create_file(create_file_request{std::string("fds_helloworld"),
false},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp
= r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty());
- }
-
- // create an exist-file for write
- {
- std::cout << "create an exist file for write: " <<
files[total_files - 1] << std::endl;
- s->create_file(create_file_request{std::string(files[total_files -
1]), false},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp
= r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_EQ(FDSClientTest::f1.md5,
cf_resp.file_handle->get_md5sum());
-
- cf_resp.file_handle
- ->upload(upload_request{FDSClientTest::f2.filename},
- lpc_btest,
- [&u_resp](const upload_response &r) { u_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, u_resp.err);
- ASSERT_EQ(FDSClientTest::f2.length,
cf_resp.file_handle->get_size());
- ASSERT_EQ(FDSClientTest::f2.md5,
cf_resp.file_handle->get_md5sum());
-
- // upload an non-exist local file
- cf_resp.file_handle
- ->upload(upload_request{"joke_file"},
- lpc_btest,
- [&u_resp](const upload_response &r) { u_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, u_resp.err);
-
- // upload an local file which we don't have read-permission
- cf_resp.file_handle
- ->upload(upload_request{"/root/.profile"},
- lpc_btest,
- [&u_resp](const upload_response &r) { u_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, u_resp.err);
- }
- }
-
- // then test list files
- {
- std::cout << "test ls files" << std::endl;
-
- // list the root
- std::cout << "list the test root" << std::endl;
- std::vector<ls_entry> root = {
- {"fdstest1", true}, {"fdstest2", true}, {"fdstest3", false},
{"fds_rootfile", false}};
- std::sort(root.begin(), root.end(), entry_cmp);
-
- s->list_dir(ls_request{"/fdstest"},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OK, l_resp.err);
- std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp);
- entry_vec_eq(root, *l_resp.entries);
-
- // list the fdstest1
- std::cout << "list the fdstest1" << std::endl;
- std::vector<ls_entry> fdstest1 = {{"test1", true}, {"test2", true}};
- std::sort(fdstest1.begin(), fdstest1.end(), entry_cmp);
-
- s->list_dir(ls_request{"/fdstest1"},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp);
- entry_vec_eq(fdstest1, *l_resp.entries);
-
- // list the fdstest1/test2
- std::cout << "list the fdstest1/test2" << std::endl;
- std::vector<ls_entry> fdstest1_slash_test2 = {{"test1", false},
{"test2", false}};
- std::sort(fdstest1_slash_test2.begin(), fdstest1_slash_test2.end(),
entry_cmp);
-
- s->list_dir(ls_request{"/fdstest1/test2"},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp);
- entry_vec_eq(fdstest1_slash_test2, *l_resp.entries);
-
- // list a non-exist dir
- std::cout << "list a non-exist dir /fds_hehe" << std::endl;
- s->list_dir(ls_request{"/fds_hehe"},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, l_resp.err);
-
- // list a regular file
- std::cout << "list a regular file /fds_rootfile" << std::endl;
- s->list_dir(ls_request{"/fds_rootfile"},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_INVALID_PARAMETERS, l_resp.err);
- }
-
- // then test download files
- {
- std::cout << "test download file, don't ignore metadata" << std::endl;
- for (int i = 0; i < total_files - 1; ++i) {
- std::cout << "test download " << files[i] << std::endl;
- s->create_file(create_file_request{files[i], false},
- lpc_btest,
- [&cf_resp](const create_file_response &resp) {
cf_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- ASSERT_EQ(f1.length, cf_resp.file_handle->get_size());
- ASSERT_EQ(f1.md5, cf_resp.file_handle->get_md5sum());
-
- cf_resp.file_handle
- ->download(download_request{local_file_for_download, 0, -1},
- lpc_btest,
- [&d_resp](const download_response &resp) { d_resp =
resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OK, d_resp.err);
- ASSERT_EQ(cf_resp.file_handle->get_size(), d_resp.downloaded_size);
- ASSERT_EQ(cf_resp.file_handle->get_md5sum(), d_resp.file_md5);
- file_eq_compare(f1.filename, local_file_for_download);
- }
-
- std::cout << "test download file, ignore metadata: " <<
files[total_files - 1] << std::endl;
- s->create_file(create_file_request{files[total_files - 1], true},
- lpc_btest,
- [&cf_resp](const create_file_response &resp) { cf_resp
= resp; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty());
-
- cf_resp.file_handle
- ->download(download_request{local_file_for_download, 0, -1},
- lpc_btest,
- [&d_resp](const download_response &resp) { d_resp =
resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(f2.length, cf_resp.file_handle->get_size());
- ASSERT_EQ(f2.md5, cf_resp.file_handle->get_md5sum());
- file_eq_compare(f2.filename, local_file_for_download);
-
- std::cout << "test partitial download " << std::endl;
- cf_resp.file_handle
- ->download(download_request{local_file_for_download, 32, 32},
- lpc_btest,
- [&d_resp](const download_response &resp) { d_resp =
resp; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, d_resp.err);
- ASSERT_EQ(32, d_resp.downloaded_size);
- {
- std::shared_ptr<FILE> f(fopen("tmp_generate", "wb"), [](FILE *p) {
fclose(p); });
- fprintf(f.get(), "%04d_this_is_a_simple_test_file\n", 1);
- }
- file_eq_compare(std::string("tmp_generate"), local_file_for_download);
- }
-
- // try to read a non-exist file
- {
- std::cout << "test try to read non-exist file" << std::endl;
- s->create_file(create_file_request{"non_exist_file", true},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp =
r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty());
-
- cf_resp.file_handle
- ->read(read_request{0, -1},
- lpc_btest,
- [&r_resp](const read_response &r) { r_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, r_resp.err);
-
- // now file handle has been synced from remote
- cf_resp.file_handle
- ->download(download_request{"local_file", 0, -1},
- lpc_btest,
- [&d_resp](const download_response &r) { d_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, d_resp.err);
- // so we expect the file doesn't create
- ASSERT_FALSE(dsn::utils::filesystem::file_exists("local_file"));
- }
-
- // try to download to a path where we can't create the file
- {
- std::cout << "test try to download to a path where we can't create the
file" << std::endl;
- s->create_file(create_file_request{files[0], false},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp =
r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- ASSERT_EQ(cf_resp.file_handle->get_size(), f1.length);
- ASSERT_EQ(cf_resp.file_handle->get_md5sum(), f1.md5);
-
- cf_resp.file_handle
- ->download(download_request{"test_dir/test_file.txt", 0, -1},
- lpc_btest,
- [&d_resp](const download_response &r) { d_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, d_resp.err);
- ASSERT_EQ(0, d_resp.downloaded_size);
-
- cf_resp.file_handle
- ->download(download_request{"/root/.profile", 0, -1},
- lpc_btest,
- [&d_resp](const download_response &r) { d_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, d_resp.err);
- ASSERT_EQ(0, d_resp.downloaded_size);
- }
-
- // try to do write/read
- {
- std::cout << "test read write operation" << std::endl;
- s->create_file(create_file_request{files[0], false},
- lpc_btest,
- [&cf_resp](const create_file_response &r) { cf_resp =
r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- ASSERT_EQ(cf_resp.file_handle->get_size(), f1.length);
- ASSERT_EQ(cf_resp.file_handle->get_md5sum(), f1.md5);
-
- const char *test_buffer = "1234567890qwertyuiopasdfghjklzxcvbnm";
- int length = strlen(test_buffer);
- dsn::blob bb(test_buffer, 0, length);
-
- cf_resp.file_handle
- ->write(write_request{bb},
- lpc_btest,
- [&w_resp](const write_response &w) { w_resp = w; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, w_resp.err);
- ASSERT_EQ(length, w_resp.written_size);
- ASSERT_EQ(length, cf_resp.file_handle->get_size());
- ASSERT_NE(f1.md5, cf_resp.file_handle->get_md5sum());
-
- std::cout << "test read just written value" << std::endl;
- cf_resp.file_handle
- ->read(read_request{0, -1},
- lpc_btest,
- [&r_resp](const read_response &r) { r_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, r_resp.err);
- ASSERT_EQ(length, r_resp.buffer.length());
- ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer,
length));
-
- // partitial read
- cf_resp.file_handle
- ->read(read_request{5, 10},
- lpc_btest,
- [&r_resp](const read_response &r) { r_resp = r; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, r_resp.err);
- ASSERT_EQ(10, r_resp.buffer.length());
- ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer + 5,
10));
- }
-
- // then test remove path
- {
- // test remove_path
- {
- std::cout << "Test remove non-empty path with recusive = false" <<
std::endl;
- s->remove_path(remove_path_request{std::string(prefix_path),
false},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(rem_resp.err, ERR_DIR_NOT_EMPTY);
-
- std::cout << "Test remove non-empty path with recusive = true" <<
std::endl;
- s->remove_path(remove_path_request{std::string(prefix_path), true},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(rem_resp.err, ERR_OK);
-
- std::cout << "Test remove a common path" << std::endl;
- for (total_files = 0; files[total_files]; total_files++) {
- std::string filename = files[total_files];
- // file under prefix_path already removed
- if (filename.find(prefix_path) == std::string::npos) {
- // remove a single file with recusive = true/false
- bool recursive = ((total_files % 2) == 0);
- s->remove_path(
- remove_path_request{std::string(files[total_files]),
recursive},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(rem_resp.err, ERR_OK);
- }
- s->create_file(create_file_request{files[total_files], false},
- lpc_btest,
- [&cf_resp](const create_file_response &resp) {
cf_resp = resp; },
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty());
- }
- }
-
- // test list_dir that total file/dir count greater than 1000
- {
- int total_file_cnt = 30;
- std::cout << "upload " << total_file_cnt << " to fds server" <<
std::endl;
- std::string prefix = "/fdstest_prefix";
- std::string file_name_prefix = "file_";
- std::vector<ls_entry> t_entries;
-
- // first clean in case last test failed
- std::cout << "first clean the dir in case old test failed" <<
std::endl;
- s->remove_path(remove_path_request{std::string(prefix), true},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
-
- // generate total_file_cnt files
- std::cout << "then start to upload files" << std::endl;
- for (int i = 1; i <= total_file_cnt; i++) {
- std::string filename = file_name_prefix + std::to_string(i);
- t_entries.emplace_back(ls_entry{std::string(filename), false});
-
- s->create_file(create_file_request{std::string(prefix + "/" +
filename), true},
- lpc_btest,
- [&cf_resp](const create_file_response &r) {
cf_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(cf_resp.err, dsn::ERR_OK);
-
- cf_resp.file_handle
- ->upload(upload_request{FDSClientTest::f1.filename},
- lpc_btest,
- [&u_resp](const upload_response &r) { u_resp = r;
},
- nullptr)
- ->wait();
-
- ASSERT_EQ(dsn::ERR_OK, u_resp.err);
- ASSERT_EQ(FDSClientTest::f1.length,
cf_resp.file_handle->get_size());
- ASSERT_EQ(FDSClientTest::f1.md5,
cf_resp.file_handle->get_md5sum());
- }
- l_resp.entries->clear();
- s->list_dir(ls_request{prefix},
- lpc_btest,
- [&l_resp](const ls_response &resp) { l_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(l_resp.err, ERR_OK);
- ASSERT_EQ(l_resp.entries->size(), total_file_cnt);
- std::sort(l_resp.entries->begin(), l_resp.entries->end(),
entry_cmp);
- std::sort(t_entries.begin(), t_entries.end(), entry_cmp);
- entry_vec_eq(t_entries, *l_resp.entries);
-
- // then remove all the file, using remove_path
- std::cout << "then remove all the files, using remove path" <<
std::endl;
- s->remove_path(remove_path_request{std::string(prefix), true},
- lpc_btest,
- [&rem_resp](const remove_path_response &resp) {
rem_resp = resp; },
- nullptr)
- ->wait();
- ASSERT_EQ(rem_resp.err, ERR_OK);
- }
- }
-}
-
-static void
-generate_file(const char *filename, unsigned long long file_size, char *block,
unsigned block_size)
-{
- int fd = ::open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR
| S_IRGRP | S_IWGRP);
- ASSERT_GT(fd, 0) << utils::safe_strerror(errno);
- for (unsigned long long i = 0; i < file_size;) {
- int batch_size = (file_size - i);
- if (batch_size > block_size)
- batch_size = block_size;
- i += batch_size;
-
- for (int j = 0; j < batch_size; ++j) {
- block[j] = (char)rand::next_u32(0, 255);
- }
- ASSERT_EQ(batch_size, ::write(fd, block, batch_size))
- << "write file " << filename << " failed, err = " <<
utils::safe_strerror(errno);
- }
- ASSERT_EQ(0, ::close(fd)) << "close file " << filename
- << " failed, err = " <<
utils::safe_strerror(errno);
-}
-
-TEST_F(FDSClientTest, test_concurrent_upload_download)
-{
- char block[1024];
- const char *str = "test_str";
- for (int i = 0; i < 128; ++i) {
- memcpy(block + i * 8, str, 8);
- }
-
- std::shared_ptr<fds_service> _service = std::make_shared<fds_service>();
- std::vector<std::string> init_str = {server_address, access_key,
access_secret, bucket_name};
-
- if (server_address == example_server_address) {
- // user don't specify the server-address, we just return true
- return;
- }
-
- _service->initialize(init_str);
-
- DSN_DEFINE_int32(fds_concurrent_test, total_files, 64, "");
- int total_files = FLAGS_total_files;
-
- std::vector<std::string> filenames;
- filenames.reserve(total_files);
- std::vector<unsigned long> filesize;
- filesize.reserve(total_files);
- std::vector<std::string> md5;
- md5.reserve(total_files);
-
- for (int i = 0; i < total_files; ++i) {
- char index[64];
- snprintf(index, 64, "%04d", i);
- unsigned long random_size = rand::next_u64(FLAGS_min_size,
FLAGS_max_size);
- std::string filename = "randomfile" + std::string(index);
- filenames.push_back(filename);
- filesize.push_back(random_size);
- generate_file(filename.c_str(), random_size, block, 1024);
-
- std::string md5result;
- dsn::utils::filesystem::md5sum(filename, md5result);
- md5.push_back(md5result);
- }
-
- printf("start delete phase\n");
- {
- for (unsigned int i = 0; i < total_files; ++i) {
- _service
- ->remove_path(remove_path_request{filenames[i], true},
- lpc_btest,
- [i, &filenames](const remove_path_response
&resp) {
- printf("file %s delete finished, err(%s)\n",
- filenames[i].c_str(),
- resp.err.to_string());
- },
- nullptr)
- ->wait();
- }
- }
-
- printf("start upload phase\n");
- {
- std::vector<block_file_ptr> block_files;
- for (unsigned int i = 0; i < total_files; ++i) {
- create_file_response cf_resp;
- _service
- ->create_file(create_file_request{filenames[i], true},
- lpc_btest,
- [&cf_resp](const create_file_response &r) {
cf_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- block_files.push_back(cf_resp.file_handle);
- }
-
- std::vector<dsn::task_ptr> callbacks;
- for (unsigned int i = 0; i < total_files; ++i) {
- block_file_ptr p = block_files[i];
- dsn::task_ptr t =
- p->upload(upload_request{filenames[i]},
- lpc_btest,
- [p, &filenames, &filesize, &md5, i](const
upload_response &ur) {
- printf("file %s upload finished\n",
filenames[i].c_str());
- ASSERT_EQ(dsn::ERR_OK, ur.err);
- ASSERT_EQ(filesize[i], ur.uploaded_size);
- ASSERT_EQ(filesize[i], p->get_size());
- ASSERT_EQ(md5[i], p->get_md5sum());
- });
- callbacks.push_back(t);
- }
-
- for (unsigned int i = 0; i < total_files; ++i) {
- callbacks[i]->wait();
- }
- }
-
- printf("start download phase\n");
- {
- std::vector<block_file_ptr> block_files;
- for (unsigned int i = 0; i < total_files; ++i) {
- create_file_response cf_resp;
- _service
- ->create_file(create_file_request{filenames[i], true},
- lpc_btest,
- [&cf_resp](const create_file_response &r) {
cf_resp = r; },
- nullptr)
- ->wait();
- ASSERT_EQ(dsn::ERR_OK, cf_resp.err);
- ASSERT_NE(nullptr, cf_resp.file_handle.get());
- block_files.push_back(cf_resp.file_handle);
- }
-
- std::vector<dsn::task_ptr> callbacks;
- for (unsigned int i = 0; i < total_files; ++i) {
- block_file_ptr p = block_files[i];
- dsn::task_ptr t =
- p->download(download_request{filenames[i] + ".b", 0, -1},
- lpc_btest,
- [&filenames, &filesize, &md5, i, p](const
download_response &dr) {
- printf("file %s download finished\n",
filenames[i].c_str());
- ASSERT_EQ(dsn::ERR_OK, dr.err);
- ASSERT_EQ(filesize[i], dr.downloaded_size);
- ASSERT_EQ(filesize[i], p->get_size());
- ASSERT_EQ(md5[i], p->get_md5sum());
- });
- callbacks.push_back(t);
- }
-
- for (unsigned int i = 0; i < total_files; ++i) {
- callbacks[i]->wait();
- }
- }
-
- printf("start cleanup phase\n");
- {
- for (unsigned int i = 0; i < total_files; ++i) {
- _service
- ->remove_path(remove_path_request{filenames[i], true},
- lpc_btest,
- [i, &filenames](const remove_path_response
&resp) {
- printf("file %s delete finished, err(%s)\n",
- filenames[i].c_str(),
- resp.err.to_string());
- },
- nullptr)
- ->wait();
- }
- }
-}
diff --git a/src/failure_detector/CMakeLists.txt
b/src/failure_detector/CMakeLists.txt
index cbeb139cc..702f02fd9 100644
--- a/src/failure_detector/CMakeLists.txt
+++ b/src/failure_detector/CMakeLists.txt
@@ -38,9 +38,7 @@ seT(MY_PROJ_SRC ${FD_THRIFT_SRCS})
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS
- PocoFoundation
- PocoJSON)
+set(MY_PROJ_LIBS "")
# Extra files that will be installed
set(MY_BINPLACES "")
diff --git a/src/failure_detector/test/config-test.ini
b/src/failure_detector/test/config-test.ini
index d1edd6385..e29d6777e 100644
--- a/src/failure_detector/test/config-test.ini
+++ b/src/failure_detector/test/config-test.ini
@@ -124,9 +124,3 @@ partitioned = true
hosts_list = localhost:22181
timeout_ms = 30000
logfile = zoolog.log
-
-[fds_concurrent_test]
-total_files = 64
-min_size = 100
-max_size = 150
-
diff --git a/src/failure_detector/test/config-whitelist-test-failed.ini
b/src/failure_detector/test/config-whitelist-test-failed.ini
index 8332ceac0..8b60870c5 100644
--- a/src/failure_detector/test/config-whitelist-test-failed.ini
+++ b/src/failure_detector/test/config-whitelist-test-failed.ini
@@ -125,9 +125,3 @@ partitioned = true
hosts_list = localhost:22181
timeout_ms = 30000
logfile = zoolog.log
-
-[fds_concurrent_test]
-total_files = 64
-min_size = 100
-max_size = 150
-
diff --git a/src/failure_detector/test/config-whitelist-test.ini
b/src/failure_detector/test/config-whitelist-test.ini
index 4ee04d7af..728507e9c 100644
--- a/src/failure_detector/test/config-whitelist-test.ini
+++ b/src/failure_detector/test/config-whitelist-test.ini
@@ -125,9 +125,3 @@ partitioned = true
hosts_list = localhost:22181
timeout_ms = 30000
logfile = zoolog.log
-
-[fds_concurrent_test]
-total_files = 64
-min_size = 100
-max_size = 150
-
diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt
index 3e4d4d413..477671acb 100644
--- a/src/meta/CMakeLists.txt
+++ b/src/meta/CMakeLists.txt
@@ -37,7 +37,6 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn.failure_detector
dsn.replication.zookeeper_provider
@@ -46,11 +45,6 @@ set(MY_PROJ_LIBS
dsn_runtime
dsn_aio
zookeeper
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
hashtable
hdfs
rocksdb)
diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt
index 60f02c691..49c9e4994 100644
--- a/src/meta/test/CMakeLists.txt
+++ b/src/meta/test/CMakeLists.txt
@@ -42,7 +42,6 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn.failure_detector
dsn_dist_cmd
@@ -52,11 +51,6 @@ set(MY_PROJ_LIBS
dsn_aio
zookeeper
hashtable
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
gtest
hdfs)
diff --git a/src/meta/test/config-test.ini b/src/meta/test/config-test.ini
index d0a865656..ca5fdf2c7 100644
--- a/src/meta/test/config-test.ini
+++ b/src/meta/test/config-test.ini
@@ -129,10 +129,6 @@ args = ./block_service
type = local_service
args =
-[block_service.fds_service]
-type = fds_service
-args =
-
[duplication-group]
master-cluster = 1
slave-cluster = 2
diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt
index 005967def..fcf789101 100644
--- a/src/replica/CMakeLists.txt
+++ b/src/replica/CMakeLists.txt
@@ -62,7 +62,6 @@ set(MY_PROJ_LIBS
dsn.failure_detector
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn_nfs
dsn_dist_cmd
@@ -70,11 +69,6 @@ set(MY_PROJ_LIBS
dsn_runtime
dsn_aio
dsn_meta_server
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
rocksdb)
set(MY_BOOST_LIBS Boost::filesystem)
diff --git a/src/replica/backup/test/CMakeLists.txt
b/src/replica/backup/test/CMakeLists.txt
index 063246df0..47158ca93 100644
--- a/src/replica/backup/test/CMakeLists.txt
+++ b/src/replica/backup/test/CMakeLists.txt
@@ -26,7 +26,6 @@ set(MY_PROJ_LIBS dsn_meta_server
dsn_replication_common
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn_utils
hashtable
gtest
diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt
index cd309ba2e..83af542d1 100644
--- a/src/replica/test/CMakeLists.txt
+++ b/src/replica/test/CMakeLists.txt
@@ -39,7 +39,6 @@ set(MY_PROJ_LIBS dsn_meta_server
dsn_replication_common
dsn.block_service
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn.failure_detector
dsn_http
diff --git a/src/replica/test/replica_http_service_test.cpp
b/src/replica/test/replica_http_service_test.cpp
index d4b49667f..d6a80544c 100644
--- a/src/replica/test/replica_http_service_test.cpp
+++ b/src/replica/test/replica_http_service_test.cpp
@@ -112,7 +112,7 @@ TEST_P(replica_http_service_test, update_config_handler)
// Update config failed and value not changed.
NO_FATALS(test_update_config(
- {{"config_sync_interval_ms", "10"}, {"fds_write_limit_rate", "50"}},
+ {{"config_sync_interval_ms", "10"},
{"hdfs_write_limit_rate_mb_per_sec", "50"}},
R"({"update_status":"ERR_INVALID_PARAMETERS: there should be exactly
one config to be updated once"})"
"\n"));
NO_FATALS(test_check_config("config_sync_interval_ms", "30000"));
diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt
index a61b75d36..ff48c181e 100644
--- a/src/server/CMakeLists.txt
+++ b/src/server/CMakeLists.txt
@@ -32,7 +32,6 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn_client
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service
dsn.failure_detector
dsn.replication.zookeeper_provider
@@ -42,14 +41,6 @@ set(MY_PROJ_LIBS
pegasus_base
pegasus_client_static
event
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
- PocoCrypto
- PocoUtil
- PocoXML
hashtable
)
diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt
index 4c3ea8a83..e9b9200f3 100644
--- a/src/server/test/CMakeLists.txt
+++ b/src/server/test/CMakeLists.txt
@@ -41,7 +41,6 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn_client
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service
dsn.failure_detector
dsn.replication.zookeeper_provider
@@ -50,11 +49,6 @@ set(MY_PROJ_LIBS
RocksDB::rocksdb
pegasus_client_static
event
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
pegasus_base
gtest
gmock
diff --git a/src/server/test/config.ini b/src/server/test/config.ini
index 1ec547264..fe4bdb907 100644
--- a/src/server/test/config.ini
+++ b/src/server/test/config.ini
@@ -42,10 +42,6 @@ logging_factory_name = dsn::tools::simple_logger
;logging_factory_name = dsn::tools::screen_logger
logging_flush_on_exit = true
-[block_service.fds_service]
-type = fds_service
-args = <server-address> <secret-key> <secret-access> <bucket-name>
-
[block_service.local_service]
type = local_service
args =
diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt
index f35ead8e8..909fda9ab 100644
--- a/src/shell/CMakeLists.txt
+++ b/src/shell/CMakeLists.txt
@@ -36,19 +36,10 @@ set(MY_PROJ_LIBS
dsn_client
dsn_utils
dsn.block_service.local
- dsn.block_service.fds
dsn.block_service.hdfs
dsn.block_service
dsn.failure_detector
pegasus_client_static
- galaxy-fds-sdk-cpp
- PocoNet
- PocoFoundation
- PocoNetSSL
- PocoJSON
- PocoCrypto
- PocoUtil
- PocoXML
pegasus_geo_lib
RocksDB::rocksdb
absl::flat_hash_set
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index bbe99ece7..e95e24f83 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -220,54 +220,6 @@ ExternalProject_Add(libevent
DOWNLOAD_NO_PROGRESS true
)
-ExternalProject_Add(poco
- URL ${OSS_URL_PREFIX}/poco-1.11.1-release.tar.gz
-
https://github.com/pocoproject/poco/archive/refs/tags/poco-1.11.1-release.tar.gz
- URL_MD5 a96210b60a675c4a8183ad1f2099d549
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
- -DENABLE_MONGODB=OFF
- -DENABLE_PDF=OFF
- -DENABLE_DATA=OFF
- -DENABLE_DATA_SQLITE=OFF
- -DENABLE_DATA_MYSQL=OFF
- -DENABLE_DATA_ODBC=OFF
- -DENABLE_SEVENZIP=OFF
- -DENABLE_ZIP=OFF
- -DENABLE_APACHECONNECTOR=OFF
- -DENABLE_CPPPARSER=OFF
- -DENABLE_POCODOC=OFF
- -DENABLE_PAGECOMPILER=OFF
- -DENABLE_PAGECOMPILER_FILE2PAGE=OFF
- -DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
- -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
- DEPENDS boost
- DOWNLOAD_EXTRACT_TIMESTAMP true
- DOWNLOAD_NO_PROGRESS true
-)
-
-ExternalProject_Add(fds
- URL ${OSS_URL_PREFIX}/fds-1.0.0.tar.gz
-
https://github.com/XiaoMi/galaxy-fds-sdk-cpp/archive/refs/tags/v1.0.0.tar.gz
- URL_MD5 f7e0f86534f7b15c2a9b349f5cb45503
- PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_fds_for_macos.patch
- CMAKE_ARGS -DPOCO_INCLUDE=${TP_OUTPUT}/include
- -DMACOS_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
- -DPOCO_LIB=${TP_OUTPUT}/lib
- -DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
- -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
- INSTALL_COMMAND cp libgalaxy-fds-sdk-cpp.a ${TP_OUTPUT}/lib
- COMMAND rm -rf ${TP_OUTPUT}/include/fds
- COMMAND cp -r include fds-include
- COMMAND mv fds-include ${TP_OUTPUT}/include/fds # install fds headers
into a stand-alone directory
- UPDATE_COMMAND "" # do not update
- DEPENDS googletest poco boost
- BUILD_IN_SOURCE 1
- DOWNLOAD_EXTRACT_TIMESTAMP true
- DOWNLOAD_NO_PROGRESS true
-)
-
ExternalProject_Add(fmt
URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz
URL_MD5 0d41a16f1b3878d44e6fd7ff1f6cc45a
diff --git a/thirdparty/fix_fds_for_macos.patch
b/thirdparty/fix_fds_for_macos.patch
deleted file mode 100644
index 5cebc13e3..000000000
--- a/thirdparty/fix_fds_for_macos.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index d2d5f09..6de3158 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -11,6 +11,9 @@ add_library(galaxy-fds-sdk-cpp ${DIR_SRCS})
- target_include_directories(galaxy-fds-sdk-cpp PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include)
-
- include_directories(${POCO_INCLUDE})
-+if (APPLE)
-+ include_directories(${MACOS_OPENSSL_INCLUDE_DIR})
-+endif()
- link_directories(${POCO_LIB})
-
- option(WITH_TEST "enable building test" OFF)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
