This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new c65c39af3 refactor(misc): use rocksdb API to read/write file (#1625)
c65c39af3 is described below
commit c65c39af34071a4de28a0507098b5306b212afa1
Author: Yingchun Lai <[email protected]>
AuthorDate: Wed Sep 27 08:01:22 2023 -0500
refactor(misc): use rocksdb API to read/write file (#1625)
https://github.com/apache/incubator-pegasus/issues/887
There is no functional changes, but only refactor the files read/write
method
of some unit tests.
---
src/block_service/test/CMakeLists.txt | 1 +
.../test/block_service_manager_test.cpp | 68 ++++++++++------------
src/replica/bulk_load/test/CMakeLists.txt | 3 +-
.../bulk_load/test/replica_bulk_loader_test.cpp | 59 +++++++------------
.../test/load_from_private_log_test.cpp | 64 +++++++++++++-------
src/replica/test/CMakeLists.txt | 3 +-
src/replica/test/mutation_log_test.cpp | 66 ++++++++-------------
src/test/function_test/bulk_load/CMakeLists.txt | 4 +-
src/test_util/CMakeLists.txt | 2 +-
src/test_util/test_util.cpp | 21 +++++++
src/test_util/test_util.h | 11 +++-
src/utils/test/CMakeLists.txt | 4 +-
12 files changed, 160 insertions(+), 146 deletions(-)
diff --git a/src/block_service/test/CMakeLists.txt
b/src/block_service/test/CMakeLists.txt
index 537778caa..19dd06808 100644
--- a/src/block_service/test/CMakeLists.txt
+++ b/src/block_service/test/CMakeLists.txt
@@ -36,6 +36,7 @@ set(MY_PROJ_LIBS
gtest
gtest_main
hdfs
+ test_utils
rocksdb)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
diff --git a/src/block_service/test/block_service_manager_test.cpp
b/src/block_service/test/block_service_manager_test.cpp
index 8ead04fd2..ef93ed196 100644
--- a/src/block_service/test/block_service_manager_test.cpp
+++ b/src/block_service/test/block_service_manager_test.cpp
@@ -15,11 +15,11 @@
// specific language governing permissions and limitations
// under the License.
+// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
#include <cstdint>
-#include <fstream>
#include <map>
#include <memory>
#include <string>
@@ -29,14 +29,16 @@
#include "block_service/local/local_service.h"
#include "block_service_mock.h"
#include "metadata_types.h"
+#include "test_util/test_util.h"
#include "utils/error_code.h"
#include "utils/filesystem.h"
+#include "utils/test_macros.h"
namespace dsn {
namespace dist {
namespace block_service {
-class block_service_manager_test : public ::testing::Test
+class block_service_manager_test : public pegasus::encrypt_data_test_base
{
public:
block_service_manager_test()
@@ -54,20 +56,6 @@ public:
PROVIDER, LOCAL_DIR, FILE_NAME, _fs.get(), download_size);
}
- void create_local_file(const std::string &file_name)
- {
- std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR,
file_name);
- utils::filesystem::create_file(whole_name);
- std::ofstream test_file;
- test_file.open(whole_name);
- test_file << "write some data.\n";
- test_file.close();
-
- _file_meta.name = whole_name;
- utils::filesystem::md5sum(whole_name, _file_meta.md5);
- utils::filesystem::file_size(whole_name, _file_meta.size);
- }
-
void create_remote_file(const std::string &file_name, int64_t size, const
std::string &md5)
{
std::string whole_file_name =
utils::filesystem::path_combine(PROVIDER, file_name);
@@ -84,8 +72,10 @@ public:
std::string FILE_NAME = "test_file";
};
-// download_file unit tests
-TEST_F(block_service_manager_test, do_download_remote_file_not_exist)
+// TODO(yingchun): ENCRYPTION: add enable encryption test.
+INSTANTIATE_TEST_CASE_P(, block_service_manager_test,
::testing::Values(false));
+
+TEST_P(block_service_manager_test, remote_file_not_exist)
{
utils::filesystem::remove_path(LOCAL_DIR);
auto fs = std::make_unique<local_service>();
@@ -93,31 +83,35 @@ TEST_F(block_service_manager_test,
do_download_remote_file_not_exist)
uint64_t download_size = 0;
error_code err = _block_service_manager.download_file(
PROVIDER, LOCAL_DIR, FILE_NAME, fs.get(), download_size);
- ASSERT_EQ(err, ERR_CORRUPTION); // file does not exist
+ ASSERT_EQ(ERR_CORRUPTION, err);
}
-TEST_F(block_service_manager_test, do_download_same_name_file)
+TEST_P(block_service_manager_test, local_file_exist)
{
- // local file exists, but md5 not matched with remote file
- create_local_file(FILE_NAME);
- create_remote_file(FILE_NAME, 2333, "md5_not_match");
- uint64_t download_size = 0;
- ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST);
- ASSERT_EQ(download_size, 0);
-}
-
-TEST_F(block_service_manager_test, do_download_file_exist)
-{
- create_local_file(FILE_NAME);
- create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5);
- uint64_t download_size = 0;
- ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST);
- ASSERT_EQ(download_size, 0);
+
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR,
FILE_NAME),
+ &_file_meta));
+ struct remote_file_info
+ {
+ int64_t size;
+ std::string md5;
+ } tests[]{
+ {2333, "bad_md5"}, // wrong size and md5
+ {2333, _file_meta.md5}, // wrong size
+ {_file_meta.size, "bad_md5"} // wrong md5
+ };
+ for (const auto &test : tests) {
+ // The remote file will be overwritten when repeatedly created.
+ create_remote_file(FILE_NAME, test.size, test.md5);
+ uint64_t download_size = 0;
+ ASSERT_EQ(ERR_PATH_ALREADY_EXIST, test_download_file(download_size));
+ ASSERT_EQ(0, download_size);
+ }
}
-TEST_F(block_service_manager_test, do_download_succeed)
+TEST_P(block_service_manager_test, do_download_succeed)
{
- create_local_file(FILE_NAME);
+
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR,
FILE_NAME),
+ &_file_meta));
create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5);
// remove local file to mock condition that file not existed
std::string file_name = utils::filesystem::path_combine(LOCAL_DIR,
FILE_NAME);
diff --git a/src/replica/bulk_load/test/CMakeLists.txt
b/src/replica/bulk_load/test/CMakeLists.txt
index 77081196c..a15861f61 100644
--- a/src/replica/bulk_load/test/CMakeLists.txt
+++ b/src/replica/bulk_load/test/CMakeLists.txt
@@ -27,7 +27,8 @@ set(MY_PROJ_LIBS dsn_meta_server
dsn_runtime
hashtable
gtest
-)
+ test_utils
+ rocksdb)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp
b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp
index df123cc0c..9eb7322a6 100644
--- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp
+++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp
@@ -17,9 +17,14 @@
#include "replica/bulk_load/replica_bulk_loader.h"
+#include <fmt/core.h>
+// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
+#include <rocksdb/env.h>
+#include <rocksdb/slice.h>
+#include <rocksdb/status.h>
#include <fstream> // IWYU pragma: keep
#include <memory>
#include <vector>
@@ -32,10 +37,11 @@
#include "replica/test/replica_test_base.h"
#include "runtime/rpc/rpc_address.h"
#include "runtime/task/task_tracker.h"
+#include "test_util/test_util.h"
#include "utils/blob.h"
#include "utils/fail_point.h"
#include "utils/filesystem.h"
-#include "utils/fmt_logging.h"
+#include "utils/test_macros.h"
namespace dsn {
namespace replication {
@@ -247,44 +253,21 @@ public:
_replica->set_primary_partition_configuration(config);
}
- void create_local_file(const std::string &file_name)
+ void create_local_metadata_file()
{
- std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR,
file_name);
- utils::filesystem::create_file(whole_name);
- std::ofstream test_file;
- test_file.open(whole_name);
- test_file << "write some data.\n";
- test_file.close();
+ NO_FATALS(pegasus::create_local_test_file(
+ utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME),
&_file_meta));
- _file_meta.name = whole_name;
- utils::filesystem::md5sum(whole_name, _file_meta.md5);
- utils::filesystem::file_size(whole_name, _file_meta.size);
- }
-
- error_code create_local_metadata_file()
- {
- create_local_file(FILE_NAME);
_metadata.files.emplace_back(_file_meta);
_metadata.file_total_size = _file_meta.size;
-
std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR,
METADATA);
- utils::filesystem::create_file(whole_name);
- std::ofstream os(whole_name.c_str(),
- (std::ofstream::out | std::ios::binary |
std::ofstream::trunc));
- if (!os.is_open()) {
- LOG_ERROR("open file {} failed", whole_name);
- return ERR_FILE_OPERATION_FAILED;
- }
-
blob bb = json::json_forwarder<bulk_load_metadata>::encode(_metadata);
- os.write((const char *)bb.data(), (std::streamsize)bb.length());
- if (os.bad()) {
- LOG_ERROR("write file {} failed", whole_name);
- return ERR_FILE_OPERATION_FAILED;
- }
- os.close();
-
- return ERR_OK;
+ auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
+ rocksdb::Slice(bb.data(),
bb.length()),
+ whole_name,
+ /* should_sync */ true);
+ ASSERT_TRUE(s.ok()) << fmt::format(
+ "write file {} failed, err = {}", whole_name, s.ToString());
}
bool validate_metadata()
@@ -549,21 +532,21 @@ TEST_F(replica_bulk_loader_test,
bulk_load_metadata_corrupt)
{
// create file can not parse as bulk_load_metadata structure
utils::filesystem::create_directory(LOCAL_DIR);
- create_local_file(METADATA);
+
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR,
METADATA),
+ &_file_meta));
std::string metadata_file_name =
utils::filesystem::path_combine(LOCAL_DIR, METADATA);
error_code ec = test_parse_bulk_load_metadata(metadata_file_name);
- ASSERT_EQ(ec, ERR_CORRUPTION);
+ ASSERT_EQ(ERR_CORRUPTION, ec);
utils::filesystem::remove_path(LOCAL_DIR);
}
TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed)
{
utils::filesystem::create_directory(LOCAL_DIR);
- error_code ec = create_local_metadata_file();
- ASSERT_EQ(ec, ERR_OK);
+ NO_FATALS(create_local_metadata_file());
std::string metadata_file_name =
utils::filesystem::path_combine(LOCAL_DIR, METADATA);
- ec = test_parse_bulk_load_metadata(metadata_file_name);
+ auto ec = test_parse_bulk_load_metadata(metadata_file_name);
ASSERT_EQ(ec, ERR_OK);
ASSERT_TRUE(validate_metadata());
utils::filesystem::remove_path(LOCAL_DIR);
diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp
b/src/replica/duplication/test/load_from_private_log_test.cpp
index 342669f63..2782f55f1 100644
--- a/src/replica/duplication/test/load_from_private_log_test.cpp
+++ b/src/replica/duplication/test/load_from_private_log_test.cpp
@@ -15,16 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-#include <boost/filesystem/path.hpp>
-#include <boost/system/error_code.hpp>
+// IWYU pragma: no_include <ext/alloc_traits.h>
#include <fcntl.h>
#include <fmt/core.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
+// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
+#include <rocksdb/status.h>
+#include <stdint.h>
#include <sys/types.h>
-#include <unistd.h>
+#include "aio/aio_task.h"
+#include "aio/file_io.h"
#include "common/gpid.h"
#include "common/replication.codes.h"
#include "common/replication_other_types.h"
@@ -44,12 +47,14 @@
#include "runtime/task/task_tracker.h"
#include "utils/autoref_ptr.h"
#include "utils/chrono_literals.h"
+#include "utils/env.h"
#include "utils/error_code.h"
#include "utils/errors.h"
#include "utils/fail_point.h"
#include "utils/filesystem.h"
#include "utils/flags.h"
#include "utils/fmt_logging.h"
+#include "utils/ports.h"
#define BOOST_NO_CXX11_SCOPED_ENUMS
#include <boost/filesystem/operations.hpp>
@@ -58,7 +63,6 @@
#include <iterator>
#include <map>
#include <memory>
-#include <ostream>
#include <string>
#include <utility>
#include <vector>
@@ -303,42 +307,43 @@ TEST_F(load_from_private_log_test,
start_duplication_100000_4MB)
// Ensure replica_duplicator can correctly handle real-world log file
TEST_F(load_from_private_log_test, handle_real_private_log)
{
+ std::vector<std::string> log_files({"log.1.0.handle_real_private_log",
+ "log.1.0.handle_real_private_log2",
+
"log.1.0.all_loaded_are_write_empties"});
+
struct test_data
{
- std::string fname;
int puts;
int total;
gpid id;
} tests[] = {
// PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY
- {"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)},
+ {4, 6, gpid(1, 4)},
// EMPTY, PUT, EMPTY
- {"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)},
+ {1, 2, gpid(1, 4)},
// EMPTY, EMPTY, EMPTY
- {"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)},
+ {0, 2, gpid(1, 5)},
};
- for (auto tt : tests) {
+ ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data));
+ for (int i = 0; i < log_files.size(); i++) {
// reset replica to specified gpid
duplicator.reset(nullptr);
- _replica = create_mock_replica(stub.get(), tt.id.get_app_id(),
tt.id.get_partition_index());
+ _replica = create_mock_replica(
+ stub.get(), tests[i].id.get_app_id(),
tests[i].id.get_partition_index());
// Update '_log_dir' to the corresponding replica created above.
_log_dir = _replica->dir();
ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir;
// Copy the log file to '_log_dir'
- boost::filesystem::path file(tt.fname);
- ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname;
- boost::system::error_code ec;
- boost::filesystem::copy_file(
- file, _log_dir + "/log.1.0",
boost::filesystem::copy_option::overwrite_if_exists, ec);
- ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", "
<< ec.message();
+ auto s = dsn::utils::copy_file(log_files[i], _log_dir + "/log.1.0");
+ ASSERT_TRUE(s.ok()) << s.ToString();
// Start to verify.
- load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0);
+ load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total,
tests[i].id, 1, 0);
}
}
@@ -451,12 +456,27 @@ TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file)
{
{ // inject some bad data in the middle of the first file
std::string log_path = _log_dir + "/log.1.0";
- auto file_size = boost::filesystem::file_size(log_path);
- int fd = open(log_path.c_str(), O_WRONLY);
+ int64_t file_size;
+ ASSERT_TRUE(utils::filesystem::file_size(
+ log_path, dsn::utils::FileDataType::kSensitive, file_size));
+ auto wfile = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY,
0666);
+ ASSERT_NE(wfile, nullptr);
+
const char buf[] = "xxxxxx";
- auto written_size = pwrite(fd, buf, sizeof(buf), file_size / 2);
- ASSERT_EQ(written_size, sizeof(buf));
- close(fd);
+ auto buff_len = sizeof(buf);
+ auto t = ::dsn::file::write(wfile,
+ buf,
+ buff_len,
+ file_size / 2,
+ LPC_AIO_IMMEDIATE_CALLBACK,
+ nullptr,
+ [=](::dsn::error_code err, size_t n) {
+ CHECK_EQ(ERR_OK, err);
+ CHECK_EQ(buff_len, n);
+ });
+ t->wait();
+ ASSERT_EQ(ERR_OK, ::dsn::file::flush(wfile));
+ ASSERT_EQ(ERR_OK, ::dsn::file::close(wfile));
}
duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP);
diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt
index 3b82d28e1..587826baa 100644
--- a/src/replica/test/CMakeLists.txt
+++ b/src/replica/test/CMakeLists.txt
@@ -47,7 +47,8 @@ set(MY_PROJ_LIBS dsn_meta_server
zookeeper
hashtable
gtest
- test_utils)
+ test_utils
+ rocksdb)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
diff --git a/src/replica/test/mutation_log_test.cpp
b/src/replica/test/mutation_log_test.cpp
index 527c1e8d9..980a538bf 100644
--- a/src/replica/test/mutation_log_test.cpp
+++ b/src/replica/test/mutation_log_test.cpp
@@ -26,12 +26,12 @@
#include "replica/mutation_log.h"
+#include <fcntl.h>
// IWYU pragma: no_include <ext/alloc_traits.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
-#include <stdio.h>
#include <sys/types.h>
#include <cstdint>
#include <iostream>
@@ -39,6 +39,7 @@
#include <unordered_map>
#include "aio/aio_task.h"
+#include "aio/file_io.h"
#include "backup_types.h"
#include "common/replication.codes.h"
#include "consensus_types.h"
@@ -53,6 +54,7 @@
#include "utils/binary_writer.h"
#include "utils/blob.h"
#include "utils/defer.h"
+#include "utils/env.h"
#include "utils/fail_point.h"
#include "utils/filesystem.h"
#include "utils/flags.h"
@@ -66,43 +68,26 @@ class message_ex;
using namespace ::dsn;
using namespace ::dsn::replication;
-static void copy_file(const char *from_file, const char *to_file, int64_t
to_size = -1)
-{
- int64_t from_size;
- ASSERT_TRUE(dsn::utils::filesystem::file_size(from_file, from_size));
- ASSERT_LE(to_size, from_size);
- FILE *from = fopen(from_file, "rb");
- ASSERT_TRUE(from != nullptr);
- FILE *to = fopen(to_file, "wb");
- ASSERT_TRUE(to != nullptr);
- if (to_size == -1)
- to_size = from_size;
- if (to_size > 0) {
- std::unique_ptr<char[]> buf(new char[to_size]);
- auto n = fread(buf.get(), 1, to_size, from);
- ASSERT_EQ(to_size, n);
- n = fwrite(buf.get(), 1, to_size, to);
- ASSERT_EQ(to_size, n);
- }
- int r = fclose(from);
- ASSERT_EQ(0, r);
- r = fclose(to);
- ASSERT_EQ(0, r);
-}
-
static void overwrite_file(const char *file, int offset, const void *buf, int
size)
{
- FILE *f = fopen(file, "r+b");
- ASSERT_TRUE(f != nullptr);
- int r = fseek(f, offset, SEEK_SET);
- ASSERT_EQ(0, r);
- size_t n = fwrite(buf, 1, size, f);
- ASSERT_EQ(size, n);
- r = fclose(f);
- ASSERT_EQ(0, r);
+ auto wfile = file::open(file, O_RDWR | O_CREAT | O_BINARY, 0666);
+ ASSERT_NE(wfile, nullptr);
+ auto t = ::dsn::file::write(wfile,
+ (const char *)buf,
+ size,
+ offset,
+ LPC_AIO_IMMEDIATE_CALLBACK,
+ nullptr,
+ [=](::dsn::error_code err, size_t n) {
+ CHECK_EQ(ERR_OK, err);
+ CHECK_EQ(size, n);
+ });
+ t->wait();
+ ASSERT_EQ(ERR_OK, file::flush(wfile));
+ ASSERT_EQ(ERR_OK, file::close(wfile));
}
-TEST(replication, log_file)
+TEST(replication_test, log_file)
{
replica_log_info_map mdecrees;
gpid gpid(1, 0);
@@ -198,7 +183,7 @@ TEST(replication, log_file)
// bad file data: empty file
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.0"));
- copy_file(fpath.c_str(), "log.1.0", 0);
+ dsn::utils::copy_file_by_size(fpath, "log.1.0", 0);
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.0"));
lf = log_file::open_read("log.1.0", err);
ASSERT_TRUE(lf == nullptr);
@@ -208,7 +193,7 @@ TEST(replication, log_file)
// bad file data: incomplete log_block_header
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.1"));
- copy_file(fpath.c_str(), "log.1.1", sizeof(log_block_header) - 1);
+ dsn::utils::copy_file_by_size(fpath, "log.1.1", sizeof(log_block_header) -
1);
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.1"));
lf = log_file::open_read("log.1.1", err);
ASSERT_TRUE(lf == nullptr);
@@ -218,7 +203,7 @@ TEST(replication, log_file)
// bad file data: bad log_block_header (magic = 0xfeadbeef)
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.2"));
- copy_file(fpath.c_str(), "log.1.2");
+ dsn::utils::copy_file_by_size(fpath, "log.1.2");
int32_t bad_magic = 0xfeadbeef;
overwrite_file("log.1.2", FIELD_OFFSET(log_block_header, magic),
&bad_magic, sizeof(bad_magic));
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.2"));
@@ -230,7 +215,7 @@ TEST(replication, log_file)
// bad file data: bad log_block_header (crc check failed)
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.3"));
- copy_file(fpath.c_str(), "log.1.3");
+ dsn::utils::copy_file_by_size(fpath, "log.1.3");
int32_t bad_crc = 0;
overwrite_file("log.1.3", FIELD_OFFSET(log_block_header, body_crc),
&bad_crc, sizeof(bad_crc));
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.3"));
@@ -242,14 +227,13 @@ TEST(replication, log_file)
// bad file data: incomplete block body
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4"));
- copy_file(fpath.c_str(), "log.1.4", sizeof(log_block_header) + 1);
+ dsn::utils::copy_file_by_size(fpath, "log.1.4", sizeof(log_block_header) +
1);
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4"));
lf = log_file::open_read("log.1.4", err);
ASSERT_TRUE(lf == nullptr);
ASSERT_EQ(ERR_INCOMPLETE_DATA, err);
ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4"));
ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4.removed"));
- ASSERT_TRUE(dsn::utils::filesystem::rename_path("log.1.4.removed",
"log.1.4"));
// read the file for test
offset = 100;
@@ -259,7 +243,7 @@ TEST(replication, log_file)
ASSERT_EQ(1, lf->index());
ASSERT_EQ(100, lf->start_offset());
int64_t sz;
- ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, sz));
+ ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath,
dsn::utils::FileDataType::kSensitive, sz));
ASSERT_EQ(lf->start_offset() + sz, lf->end_offset());
// read data
diff --git a/src/test/function_test/bulk_load/CMakeLists.txt
b/src/test/function_test/bulk_load/CMakeLists.txt
index a6eac0851..ea27eed37 100644
--- a/src/test/function_test/bulk_load/CMakeLists.txt
+++ b/src/test/function_test/bulk_load/CMakeLists.txt
@@ -37,8 +37,8 @@ set(MY_PROJ_LIBS
gssapi_krb5
krb5
function_test_utils
- rocksdb
- test_utils)
+ test_utils
+ rocksdb)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt
index 267c825e4..b1e7ac2df 100644
--- a/src/test_util/CMakeLists.txt
+++ b/src/test_util/CMakeLists.txt
@@ -22,6 +22,6 @@ set(MY_PROJ_NAME test_utils)
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS gtest)
+set(MY_PROJ_LIBS gtest rocksdb)
dsn_add_static_library()
diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp
index 5ab185503..0789c4678 100644
--- a/src/test_util/test_util.cpp
+++ b/src/test_util/test_util.cpp
@@ -26,12 +26,33 @@
#include "gtest/gtest-message.h"
#include "gtest/gtest-test-part.h"
#include "gtest/gtest.h"
+#include "metadata_types.h"
+#include "rocksdb/env.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
#include "runtime/api_layer1.h"
#include "utils/defer.h"
+#include "utils/env.h"
+#include "utils/error_code.h"
+#include "utils/filesystem.h"
#include "utils/fmt_logging.h"
namespace pegasus {
+void create_local_test_file(const std::string &full_name,
dsn::replication::file_meta *fm)
+{
+ ASSERT_NE(fm, nullptr);
+ auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
+ rocksdb::Slice("write some data."),
+ full_name,
+ /* should_sync */ true);
+ ASSERT_TRUE(s.ok()) << s.ToString();
+ fm->name = full_name;
+ ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5));
+ ASSERT_TRUE(dsn::utils::filesystem::file_size(
+ full_name, dsn::utils::FileDataType::kSensitive, fm->size));
+}
+
void AssertEventually(const std::function<void(void)> &f, int timeout_sec,
WaitBackoff backoff)
{
// TODO(yingchun): should use mono time
diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h
index d33775ff5..762d15436 100644
--- a/src/test_util/test_util.h
+++ b/src/test_util/test_util.h
@@ -19,12 +19,19 @@
#pragma once
+#include <gtest/gtest.h>
#include <functional>
+#include <string>
-#include "gtest/gtest.h"
#include "utils/flags.h"
#include "utils/test_macros.h"
+namespace dsn {
+namespace replication {
+class file_meta;
+} // namespace replication
+} // namespace dsn
+
DSN_DECLARE_bool(encrypt_data_at_rest);
namespace pegasus {
@@ -36,6 +43,8 @@ public:
encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); }
};
+void create_local_test_file(const std::string &full_name,
dsn::replication::file_meta *fm);
+
#define ASSERT_EVENTUALLY(expr)
\
do {
\
AssertEventually(expr);
\
diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt
index 266b9beeb..d77464c36 100644
--- a/src/utils/test/CMakeLists.txt
+++ b/src/utils/test/CMakeLists.txt
@@ -33,8 +33,8 @@ set(MY_PROJ_LIBS dsn_http
dsn_runtime
dsn_utils
gtest
- rocksdb
- test_utils)
+ test_utils
+ rocksdb)
set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
