This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new c4470fa4b refactor(nfs_test): use rocksdb API to read/write file
(#1621)
c4470fa4b is described below
commit c4470fa4b5631fda4ed9a12ef63449d5fd6de9bc
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Sep 26 04:43:11 2023 -0500
refactor(nfs_test): use rocksdb API to read/write file (#1621)
https://github.com/apache/incubator-pegasus/issues/887
There is no functional changes, but only refactor the nfs unit test.
---
src/nfs/test/main.cpp | 159 ++++++++++++++++++++++++++++++++------------------
1 file changed, 103 insertions(+), 56 deletions(-)
diff --git a/src/nfs/test/main.cpp b/src/nfs/test/main.cpp
index 445c4732f..078812992 100644
--- a/src/nfs/test/main.cpp
+++ b/src/nfs/test/main.cpp
@@ -24,13 +24,16 @@
* THE SOFTWARE.
*/
+// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
#include <stddef.h>
-#include <stdint.h>
+#include <algorithm>
+#include <cstdint>
#include <memory>
#include <string>
+#include <utility>
#include <vector>
#include "aio/aio_task.h"
@@ -40,7 +43,9 @@
#include "runtime/rpc/rpc_address.h"
#include "runtime/task/task_code.h"
#include "runtime/tool_api.h"
+#include "test_util/test_util.h"
#include "utils/autoref_ptr.h"
+#include "utils/env.h"
#include "utils/error_code.h"
#include "utils/filesystem.h"
#include "utils/threadpool_code.h"
@@ -54,36 +59,55 @@ struct aio_result
size_t sz;
};
-TEST(nfs, basic)
+class nfs_test : public pegasus::encrypt_data_test_base
{
- std::unique_ptr<dsn::nfs_node> nfs(dsn::nfs_node::create());
+};
+
+// TODO(yingchun): ENCRYPTION: add enable encryption test.
+INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false));
+
+TEST_P(nfs_test, basic)
+{
+ auto nfs = dsn::nfs_node::create();
nfs->start();
nfs->register_async_rpc_handler_for_test();
dsn::gpid fake_pid = gpid(1, 0);
- utils::filesystem::remove_path("nfs_test_dir");
- utils::filesystem::remove_path("nfs_test_dir_copy");
-
- ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir"));
- ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy"));
-
- ASSERT_TRUE(utils::filesystem::create_directory("nfs_test_dir"));
- ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir"));
+ // Prepare the destination directory.
+ const std::string kDstDir = "nfs_test_dir";
+ ASSERT_TRUE(utils::filesystem::remove_path(kDstDir));
+ ASSERT_FALSE(utils::filesystem::directory_exists(kDstDir));
+ ASSERT_TRUE(utils::filesystem::create_directory(kDstDir));
+ ASSERT_TRUE(utils::filesystem::directory_exists(kDstDir));
+
+ // Prepare the source files information.
+ std::vector<std::string> kSrcFilenames({"nfs_test_file1",
"nfs_test_file2"});
+ std::vector<int64_t> src_file_sizes;
+ std::vector<std::string> src_file_md5s;
+ for (const auto &src_filename : kSrcFilenames) {
+ int64_t file_size;
+ ASSERT_TRUE(utils::filesystem::file_size(
+ src_filename, dsn::utils::FileDataType::kSensitive, file_size));
+ src_file_sizes.push_back(file_size);
+ std::string src_file_md5;
+ ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_filename,
src_file_md5));
+ src_file_md5s.emplace_back(std::move(src_file_md5));
+ }
+ // copy files to the destination directory.
{
- // copy nfs_test_file1 nfs_test_file2 nfs_test_dir
-
ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1"));
-
ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2"));
-
- std::vector<std::string> files{"nfs_test_file1", "nfs_test_file2"};
+ // The destination directory is empty before copying.
+ std::vector<std::string> dst_filenames;
+ ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames,
true));
+ ASSERT_TRUE(dst_filenames.empty());
aio_result r;
dsn::aio_task_ptr t =
nfs->copy_remote_files(dsn::rpc_address("localhost", 20101),
"default",
".",
- files,
+ kSrcFilenames,
"default",
- "nfs_test_dir",
+ kDstDir,
fake_pid,
false,
false,
@@ -100,32 +124,32 @@ TEST(nfs, basic)
ASSERT_EQ(ERR_OK, r.err);
ASSERT_EQ(r.sz, t->get_transferred_size());
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1"));
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2"));
-
- int64_t sz1, sz2;
- ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file1", sz1));
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz2));
- ASSERT_EQ(sz1, sz2);
- ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file2", sz1));
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz2));
- ASSERT_EQ(sz1, sz2);
+ // The destination files equal to the source files after copying.
+ ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames,
true));
+ std::sort(dst_filenames.begin(), dst_filenames.end());
+ ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size());
+ int i = 0;
+ for (const auto &dst_filename : dst_filenames) {
+ int64_t file_size;
+ ASSERT_TRUE(utils::filesystem::file_size(
+ dst_filename, dsn::utils::FileDataType::kSensitive,
file_size));
+ ASSERT_EQ(src_file_sizes[i], file_size);
+ std::string file_md5;
+ ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename,
file_md5));
+ ASSERT_EQ(src_file_md5s[i], file_md5);
+ i++;
+ }
}
+ // copy files to the destination directory, files will be overwritten.
{
- // copy files again, overwrite
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1"));
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2"));
-
- std::vector<std::string> files{"nfs_test_file1", "nfs_test_file2"};
-
aio_result r;
dsn::aio_task_ptr t =
nfs->copy_remote_files(dsn::rpc_address("localhost", 20101),
"default",
".",
- files,
+ kSrcFilenames,
"default",
- "nfs_test_dir",
+ kDstDir,
fake_pid,
true,
false,
@@ -141,22 +165,42 @@ TEST(nfs, basic)
ASSERT_EQ(r.err, t->error());
ASSERT_EQ(ERR_OK, r.err);
ASSERT_EQ(r.sz, t->get_transferred_size());
+
// this is only true for simulator
if (dsn::tools::get_current_tool()->name() == "simulator") {
ASSERT_EQ(1, t->get_count());
}
+
+ // The destination files equal to the source files after overwrite
copying.
+ std::vector<std::string> dst_filenames;
+ ASSERT_TRUE(utils::filesystem::get_subfiles(kDstDir, dst_filenames,
true));
+ std::sort(dst_filenames.begin(), dst_filenames.end());
+ ASSERT_EQ(kSrcFilenames.size(), dst_filenames.size());
+ int i = 0;
+ for (const auto &dst_filename : dst_filenames) {
+ int64_t file_size;
+ ASSERT_TRUE(utils::filesystem::file_size(
+ dst_filename, dsn::utils::FileDataType::kSensitive,
file_size));
+ ASSERT_EQ(src_file_sizes[i], file_size);
+ std::string file_md5;
+ ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename,
file_md5));
+ ASSERT_EQ(src_file_md5s[i], file_md5);
+ i++;
+ }
}
+ // copy files from kDstDir to kNewDstDir.
{
- // copy nfs_test_dir nfs_test_dir_copy
- ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy"));
+ const std::string kNewDstDir = "nfs_test_dir_copy";
+ ASSERT_TRUE(utils::filesystem::remove_path(kNewDstDir));
+ ASSERT_FALSE(utils::filesystem::directory_exists(kNewDstDir));
aio_result r;
dsn::aio_task_ptr t =
nfs->copy_remote_directory(dsn::rpc_address("localhost", 20101),
"default",
- "nfs_test_dir",
+ kDstDir,
"default",
- "nfs_test_dir_copy",
+ kNewDstDir,
fake_pid,
false,
false,
@@ -173,22 +217,25 @@ TEST(nfs, basic)
ASSERT_EQ(ERR_OK, r.err);
ASSERT_EQ(r.sz, t->get_transferred_size());
- ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir_copy"));
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file1"));
-
ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file2"));
-
- std::vector<std::string> sub1, sub2;
- ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir", sub1,
true));
- ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir_copy", sub2,
true));
- ASSERT_EQ(sub1.size(), sub2.size());
-
- int64_t sz1, sz2;
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz1));
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file1",
sz2));
- ASSERT_EQ(sz1, sz2);
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz1));
-
ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file2",
sz2));
- ASSERT_EQ(sz1, sz2);
+ // The kNewDstDir will be created automatically.
+ ASSERT_TRUE(utils::filesystem::directory_exists(kNewDstDir));
+
+ std::vector<std::string> new_dst_filenames;
+ ASSERT_TRUE(utils::filesystem::get_subfiles(kNewDstDir,
new_dst_filenames, true));
+ std::sort(new_dst_filenames.begin(), new_dst_filenames.end());
+ ASSERT_EQ(kSrcFilenames.size(), new_dst_filenames.size());
+
+ int i = 0;
+ for (const auto &new_dst_filename : new_dst_filenames) {
+ int64_t file_size;
+ ASSERT_TRUE(utils::filesystem::file_size(
+ new_dst_filename, dsn::utils::FileDataType::kSensitive,
file_size));
+ ASSERT_EQ(src_file_sizes[i], file_size);
+ std::string file_md5;
+ ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(new_dst_filename,
file_md5));
+ ASSERT_EQ(src_file_md5s[i], file_md5);
+ i++;
+ }
}
nfs->stop();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
