Repository: kudu Updated Branches: refs/heads/master f329e089f -> fc98b1ae7
external minicluster: expand EMC dir usage In order to test different disk configurations, it is becoming increasingly important to have end-to-end testing with nodes backed by multiple directories. External miniclusters by default use a single directory for each daemon's data (i.e. wals and data dirs fall under a single /cluster/daemon/data directory). This patch adds multi-directory support via a new 'num_data_dirs' parameter to ExternalMiniClusterOptions. Additionally, a 'wal_dir' parameter is added to ExternalDaemonOptions to separate the wal location from the data directories. If 'num_data_dirs' is greater than 1, each daemon will generate multiple paths, appending each with a numeric suffix, up to the number specified. E.g. EMCs that would have used the path /cluster/data, if specifying 'num_data_dirs' as 3, will now generate multiple data directories /cluster/data-0, /cluster/data-1, /cluster/data-2. The wal will be added to /cluster/wal. The new test multidir-cluster-itest demonstrates this. This test has been run via dist-test 2000 with no flakes. Results here: http://dist-test.cloudera.org/job?job_id=awong.1496701042.24862 Change-Id: Id2f5def6980ad394c8558ad97ba830f1b0257332 Reviewed-on: http://gerrit.cloudera.org:8080/6845 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon <[email protected]> Reviewed-by: Mike Percy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc98b1ae Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc98b1ae Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc98b1ae Branch: refs/heads/master Commit: fc98b1ae77f6fd869863191fcb7fa5d5363d9fc5 Parents: f329e08 Author: Andrew Wong <[email protected]> Authored: Tue May 30 11:47:05 2017 -0700 Committer: Mike Percy <[email protected]> Committed: Tue Jun 6 01:38:15 2017 +0000 ---------------------------------------------------------------------- src/kudu/integration-tests/CMakeLists.txt | 1 + .../integration-tests/delete_table-itest.cc | 2 +- .../integration-tests/disk_reservation-itest.cc | 18 ++-- .../external_mini_cluster-itest-base.cc | 4 +- .../external_mini_cluster-itest-base.h | 3 +- .../integration-tests/external_mini_cluster.cc | 67 ++++++++++---- .../integration-tests/external_mini_cluster.h | 44 ++++++++-- .../external_mini_cluster_fs_inspector.cc | 16 ++-- src/kudu/integration-tests/log_verifier.cc | 6 +- .../integration-tests/master_failover-itest.cc | 13 ++- .../integration-tests/master_migration-itest.cc | 11 ++- .../integration-tests/multidir_cluster-itest.cc | 92 ++++++++++++++++++++ .../integration-tests/open-readonly-fs-itest.cc | 4 +- src/kudu/integration-tests/ts_itest-base.h | 4 +- src/kudu/integration-tests/ts_recovery-itest.cc | 5 +- 15 files changed, 230 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index f3a3cc9..07b00a1 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -81,6 +81,7 @@ ADD_KUDU_TEST_DEPENDENCIES(master_migration-itest kudu) ADD_KUDU_TEST(master_replication-itest RESOURCE_LOCK "master-rpc-ports") ADD_KUDU_TEST(master-stress-test RESOURCE_LOCK "master-rpc-ports") +ADD_KUDU_TEST(multidir_cluster-itest) ADD_KUDU_TEST(open-readonly-fs-itest) ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true) ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port") http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/delete_table-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/delete_table-itest.cc b/src/kudu/integration-tests/delete_table-itest.cc index 9c78054..9419aa7 100644 --- a/src/kudu/integration-tests/delete_table-itest.cc +++ b/src/kudu/integration-tests/delete_table-itest.cc @@ -1060,7 +1060,7 @@ TEST_F(DeleteTableITest, TestUnknownTabletsAreNotDeleted) { // Delete the master's metadata and start it back up. The tablet created // above is now unknown, but should not be deleted! cluster_->master()->Shutdown(); - ASSERT_OK(env_->DeleteRecursively(cluster_->master()->data_dir())); + ASSERT_OK(cluster_->master()->DeleteFromDisk()); ASSERT_OK(cluster_->master()->Restart()); // Give the master a chance to finish writing the new master tablet to disk http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/disk_reservation-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/disk_reservation-itest.cc b/src/kudu/integration-tests/disk_reservation-itest.cc index d30f2e5..b19ffa4 100644 --- a/src/kudu/integration-tests/disk_reservation-itest.cc +++ b/src/kudu/integration-tests/disk_reservation-itest.cc @@ -59,13 +59,15 @@ TEST_F(DiskReservationITest, TestFillMultipleDisks) { // Reserve one byte so that when we simulate 0 bytes free below, we'll start // failing requests. ts_flags.push_back("--fs_data_dirs_reserved_bytes=1"); - ts_flags.push_back(Substitute("--fs_data_dirs=$0/a,$0/b", test_dir_)); - ts_flags.push_back(Substitute("--disk_reserved_override_prefix_1_path_for_testing=$0/a", - test_dir_)); - ts_flags.push_back(Substitute("--disk_reserved_override_prefix_2_path_for_testing=$0/b", - test_dir_)); - NO_FATALS(StartCluster(ts_flags, {}, 1)); + NO_FATALS(StartCluster(ts_flags, {}, /* num_tablet_servers= */ 1, /* num_data_dirs= */ 2)); + + ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0), + "disk_reserved_override_prefix_1_path_for_testing", + cluster_->GetDataPath("ts-0", 0))); + ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0), + "disk_reserved_override_prefix_2_path_for_testing", + cluster_->GetDataPath("ts-0", 1))); TestWorkload workload(cluster_.get()); workload.set_num_replicas(1); @@ -77,10 +79,10 @@ TEST_F(DiskReservationITest, TestFillMultipleDisks) { workload.Setup(); workload.Start(); - // Simulate that /a has 0 bytes free. + // Simulate that /data-0 has 0 bytes free. ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0), "disk_reserved_override_prefix_1_bytes_free_for_testing", "0")); - // Simulate that /b has 1GB free. + // Simulate that /data-1 has 1GB free. ASSERT_OK(cluster_->SetFlag(cluster_->tablet_server(0), "disk_reserved_override_prefix_2_bytes_free_for_testing", Substitute("$0", 1L * 1024 * 1024 * 1024))); http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster-itest-base.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc index f4db17f..6b5eefb 100644 --- a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc +++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc @@ -40,8 +40,10 @@ void ExternalMiniClusterITestBase::TearDown() { void ExternalMiniClusterITestBase::StartCluster( const std::vector<std::string>& extra_ts_flags, const std::vector<std::string>& extra_master_flags, - int num_tablet_servers) { + int num_tablet_servers, + int num_data_dirs) { ExternalMiniClusterOptions opts; + opts.num_data_dirs = num_data_dirs; opts.num_tablet_servers = num_tablet_servers; opts.extra_master_flags = extra_master_flags; opts.extra_tserver_flags = extra_ts_flags; http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster-itest-base.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.h b/src/kudu/integration-tests/external_mini_cluster-itest-base.h index 87e4750..9032c1a 100644 --- a/src/kudu/integration-tests/external_mini_cluster-itest-base.h +++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.h @@ -46,7 +46,8 @@ class ExternalMiniClusterITestBase : public KuduTest { protected: void StartCluster(const std::vector<std::string>& extra_ts_flags = {}, const std::vector<std::string>& extra_master_flags = {}, - int num_tablet_servers = 3); + int num_tablet_servers = 3, + int num_data_dirs = 1); void StartClusterWithOpts(ExternalMiniClusterOptions opts); http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster.cc b/src/kudu/integration-tests/external_mini_cluster.cc index 79143ea..85f28dc 100644 --- a/src/kudu/integration-tests/external_mini_cluster.cc +++ b/src/kudu/integration-tests/external_mini_cluster.cc @@ -93,6 +93,7 @@ ExternalMiniClusterOptions::ExternalMiniClusterOptions() : num_masters(1), num_tablet_servers(1), bind_mode(kBindMode), + num_data_dirs(1), enable_kerberos(false), logtostderr(true), start_process_timeout(MonoDelta::FromSeconds(30)) { @@ -241,14 +242,38 @@ string ExternalMiniCluster::GetBinaryPath(const string& binary) const { return JoinPathSegments(daemon_bin_path_, binary); } -string ExternalMiniCluster::GetDataPath(const string& daemon_id) const { +string ExternalMiniCluster::GetLogPath(const string& daemon_id) const { CHECK(!data_root_.empty()); - return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "data"); + return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "logs"); } -string ExternalMiniCluster::GetLogPath(const string& daemon_id) const { +string ExternalMiniCluster::GetDataPath(const string& daemon_id, + boost::optional<uint32_t> dir_index) const { CHECK(!data_root_.empty()); - return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "logs"); + string data_path = "data"; + if (dir_index) { + CHECK_LT(*dir_index, opts_.num_data_dirs); + data_path = Substitute("$0-$1", data_path, dir_index.get()); + } else { + CHECK_EQ(1, opts_.num_data_dirs); + } + return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), data_path); +} + +vector<string> ExternalMiniCluster::GetDataPaths(const string& daemon_id) const { + if (opts_.num_data_dirs == 1) { + return { GetDataPath(daemon_id) }; + } + vector<string> paths; + for (uint32_t dir_index = 0; dir_index < opts_.num_data_dirs; dir_index++) { + paths.emplace_back(GetDataPath(daemon_id, dir_index)); + } + return paths; +} + +string ExternalMiniCluster::GetWalPath(const string& daemon_id) const { + CHECK(!data_root_.empty()); + return JoinPathSegments(JoinPathSegments(data_root_, daemon_id), "wal"); } namespace { @@ -270,7 +295,8 @@ Status ExternalMiniCluster::StartSingleMaster() { ExternalDaemonOptions opts(opts_.logtostderr); opts.messenger = messenger_; opts.exe = GetBinaryPath(kMasterBinaryName); - opts.data_dir = GetDataPath(daemon_id); + opts.wal_dir = GetWalPath(daemon_id); + opts.data_dirs = GetDataPaths(daemon_id); opts.log_dir = GetLogPath(daemon_id); if (FLAGS_perf_record) { opts.perf_record_filename = @@ -313,7 +339,8 @@ Status ExternalMiniCluster::StartDistributedMasters() { ExternalDaemonOptions opts(opts_.logtostderr); opts.messenger = messenger_; opts.exe = exe; - opts.data_dir = GetDataPath(daemon_id); + opts.wal_dir = GetWalPath(daemon_id); + opts.data_dirs = GetDataPaths(daemon_id); opts.log_dir = GetLogPath(daemon_id); if (FLAGS_perf_record) { opts.perf_record_filename = @@ -365,7 +392,8 @@ Status ExternalMiniCluster::AddTabletServer() { ExternalDaemonOptions opts(opts_.logtostderr); opts.messenger = messenger_; opts.exe = GetBinaryPath(kTabletServerBinaryName); - opts.data_dir = GetDataPath(daemon_id); + opts.wal_dir = GetWalPath(daemon_id); + opts.data_dirs = GetDataPaths(daemon_id); opts.log_dir = GetLogPath(daemon_id); if (FLAGS_perf_record) { opts.perf_record_filename = @@ -617,7 +645,8 @@ Status ExternalMiniCluster::SetFlag(ExternalDaemon* daemon, ExternalDaemon::ExternalDaemon(ExternalDaemonOptions opts) : messenger_(std::move(opts.messenger)), - data_dir_(std::move(opts.data_dir)), + wal_dir_(std::move(opts.wal_dir)), + data_dirs_(std::move(opts.data_dirs)), log_dir_(std::move(opts.log_dir)), perf_record_filename_(std::move(opts.perf_record_filename)), start_process_timeout_(opts.start_process_timeout), @@ -689,7 +718,7 @@ Status ExternalDaemon::StartProcess(const vector<string>& user_flags) { RETURN_NOT_OK(env_util::CreateDirsRecursively(Env::Default(), log_dir_)); // Tell the server to dump its port information so we can pick it up. - string info_path = JoinPathSegments(data_dir_, "info.pb"); + string info_path = JoinPathSegments(data_dirs_[0], "info.pb"); argv.push_back("--server_dump_info_path=" + info_path); argv.push_back("--server_dump_info_format=pb"); @@ -909,6 +938,14 @@ void ExternalDaemon::Shutdown() { perf_record_process_.reset(); } +Status ExternalDaemon::DeleteFromDisk() const { + for (const string& data_dir : data_dirs()) { + RETURN_NOT_OK(Env::Default()->DeleteRecursively(data_dir)); + } + RETURN_NOT_OK(Env::Default()->DeleteRecursively(wal_dir())); + return Status::OK(); +} + void ExternalDaemon::FlushCoverage() { #ifndef COVERAGE_BUILD return; @@ -1144,8 +1181,8 @@ Status ExternalMaster::WaitForCatalogManager() { vector<string> ExternalMaster::GetCommonFlags() const { return { - "--fs_wal_dir=" + data_dir_, - "--fs_data_dirs=" + data_dir_, + "--fs_wal_dir=" + wal_dir_, + "--fs_data_dirs=" + JoinStrings(data_dirs_, ","), "--webserver_interface=localhost", // See the in-line comment for "--ipki_server_key_size" flag in @@ -1176,8 +1213,8 @@ ExternalTabletServer::~ExternalTabletServer() { Status ExternalTabletServer::Start() { vector<string> flags; - flags.push_back("--fs_wal_dir=" + data_dir_); - flags.push_back("--fs_data_dirs=" + data_dir_); + flags.push_back("--fs_wal_dir=" + wal_dir_); + flags.push_back("--fs_data_dirs=" + JoinStrings(data_dirs_, ",")); flags.push_back(Substitute("--rpc_bind_addresses=$0:0", get_rpc_bind_address())); flags.push_back(Substitute("--local_ip_for_outbound_sockets=$0", @@ -1196,8 +1233,8 @@ Status ExternalTabletServer::Restart() { return Status::IllegalState("Tablet server cannot be restarted. Must call Shutdown() first."); } vector<string> flags; - flags.push_back("--fs_wal_dir=" + data_dir_); - flags.push_back("--fs_data_dirs=" + data_dir_); + flags.push_back("--fs_wal_dir=" + wal_dir_); + flags.push_back("--fs_data_dirs=" + JoinStrings(data_dirs_, ",")); flags.push_back("--rpc_bind_addresses=" + bound_rpc_.ToString()); flags.push_back(Substitute("--local_ip_for_outbound_sockets=$0", get_rpc_bind_address())); http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster.h b/src/kudu/integration-tests/external_mini_cluster.h index 0f8d3b3..e8d25c8 100644 --- a/src/kudu/integration-tests/external_mini_cluster.h +++ b/src/kudu/integration-tests/external_mini_cluster.h @@ -14,16 +14,18 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#ifndef KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H -#define KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H +#pragma once + +#include <sys/types.h> #include <functional> #include <map> #include <memory> #include <string> -#include <sys/types.h> #include <vector> +#include <boost/optional.hpp> + #include "kudu/client/client.h" #include "kudu/gutil/gscoped_ptr.h" #include "kudu/gutil/macros.h" @@ -110,6 +112,10 @@ struct ExternalMiniClusterOptions { // This works for unit tests, since they all end up in build/latest/bin. std::string daemon_bin_path; + // Number of data directories to be created for each daemon. + // Default: 1 + int num_data_dirs; + // Extra flags for tablet servers and masters respectively. // // In these flags, you may use the special string '${index}' which will @@ -289,7 +295,18 @@ class ExternalMiniCluster : public MiniClusterBase { // Returns the path where 'daemon_id' is expected to store its data, based on // ExternalMiniClusterOptions.data_root if it was provided, or on the // standard Kudu test directory otherwise. - std::string GetDataPath(const std::string& daemon_id) const; + // 'dir_index' is an optional numeric suffix to be added to the default path. + // If it is not specified, the cluster must be configured to use a single data dir. + std::string GetDataPath(const std::string& daemon_id, + boost::optional<uint32_t> dir_index = boost::none) const; + + // Returns paths where 'daemon_id' is expected to store its data, each with a + // numeric suffix appropriate for 'opts_.num_data_dirs' + std::vector<std::string> GetDataPaths(const std::string& daemon_id) const; + + // Returns the path where 'daemon_id' is expected to store its wal, or other + // files that reside in the wal dir. + std::string GetWalPath(const std::string& daemon_id) const; // Returns the path where 'daemon_id' is expected to store its logs, or other // files that reside in the log dir. @@ -329,7 +346,8 @@ struct ExternalDaemonOptions { bool logtostderr; std::shared_ptr<rpc::Messenger> messenger; std::string exe; - std::string data_dir; + std::string wal_dir; + std::vector<std::string> data_dirs; std::string log_dir; std::string perf_record_filename; std::vector<std::string> extra_flags; @@ -402,7 +420,17 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> { virtual void Shutdown(); - const std::string& data_dir() const { return data_dir_; } + // Delete files specified by 'wal_dir_' and 'data_dirs_'. + Status DeleteFromDisk() const WARN_UNUSED_RESULT; + + const std::string& wal_dir() const { return wal_dir_; } + + const std::string& data_dir() const { + CHECK_EQ(1, data_dirs_.size()); + return data_dirs_[0]; + } + + const std::vector<std::string>& data_dirs() const { return data_dirs_; } // Returns the log dir of the external daemon. const std::string& log_dir() const { return log_dir_; } @@ -461,7 +489,8 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> { } const std::shared_ptr<rpc::Messenger> messenger_; - const std::string data_dir_; + const std::string wal_dir_; + std::vector<std::string> data_dirs_; const std::string log_dir_; const std::string perf_record_filename_; const MonoDelta start_process_timeout_; @@ -547,4 +576,3 @@ class ExternalTabletServer : public ExternalDaemon { }; } // namespace kudu -#endif /* KUDU_INTEGRATION_TESTS_EXTERNAL_MINI_CLUSTER_H */ http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc index 3690f47..c93182b 100644 --- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc +++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc @@ -79,8 +79,8 @@ int ExternalMiniClusterFsInspector::CountFilesInDir(const string& path, } int ExternalMiniClusterFsInspector::CountWALFilesOnTS(int index) { - string data_dir = cluster_->tablet_server(index)->data_dir(); - string ts_wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName); + string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), + FsManager::kWalDirName); vector<string> tablets; CHECK_OK(ListFilesInDir(ts_wal_dir, &tablets)); int total_segments = 0; @@ -109,8 +109,8 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) { } vector<string> ExternalMiniClusterFsInspector::ListTabletsWithDataOnTS(int index) { - string data_dir = cluster_->tablet_server(index)->data_dir(); - string wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName); + string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), + FsManager::kWalDirName); vector<string> tablets; CHECK_OK(ListFilesInDir(wal_dir, &tablets)); return tablets; @@ -120,8 +120,8 @@ int ExternalMiniClusterFsInspector::CountFilesInWALDirForTS( int index, const string& tablet_id, StringPiece pattern) { - string data_dir = cluster_->tablet_server(index)->data_dir(); - string wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName); + string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), + FsManager::kWalDirName); string tablet_wal_dir = JoinPathSegments(wal_dir, tablet_id); if (!env_->FileExists(tablet_wal_dir)) { return 0; @@ -336,8 +336,8 @@ Status ExternalMiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs( Status s; MonoTime deadline = MonoTime::Now() + timeout; - string data_dir = cluster_->tablet_server(ts_index)->data_dir(); - string ts_wal_dir = JoinPathSegments(data_dir, FsManager::kWalDirName); + string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(ts_index)->wal_dir(), + FsManager::kWalDirName); string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet_id); string error_msg; http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/log_verifier.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/log_verifier.cc b/src/kudu/integration-tests/log_verifier.cc index 9a8f919..05fefd1 100644 --- a/src/kudu/integration-tests/log_verifier.cc +++ b/src/kudu/integration-tests/log_verifier.cc @@ -60,13 +60,13 @@ LogVerifier::~LogVerifier() { Status LogVerifier::OpenFsManager(ExternalTabletServer* ets, unique_ptr<FsManager>* fs) { - const string& data_dir = ets->data_dir(); FsManagerOpts fs_opts; fs_opts.read_only = true; - fs_opts.wal_path = data_dir; + fs_opts.wal_path = ets->wal_dir(); + fs_opts.data_paths = ets->data_dirs(); unique_ptr<FsManager> ret(new FsManager(Env::Default(), fs_opts)); RETURN_NOT_OK_PREPEND(ret->Open(), - Substitute("Couldn't initialize FS Manager for $0", data_dir)); + Substitute("Couldn't initialize FS Manager for $0", ets->wal_dir())); fs->swap(ret); return Status::OK(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/master_failover-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/master_failover-itest.cc b/src/kudu/integration-tests/master_failover-itest.cc index 5288e89..ed96b7f 100644 --- a/src/kudu/integration-tests/master_failover-itest.cc +++ b/src/kudu/integration-tests/master_failover-itest.cc @@ -363,8 +363,7 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) { // "Fail" a master and blow away its state completely. failed_master->Shutdown(); - string data_root = failed_master->data_dir(); - env_->DeleteRecursively(data_root); + ASSERT_OK(failed_master->DeleteFromDisk()); // Pick another master at random to serve as a basis for recovery. // @@ -382,7 +381,7 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) { "local_replica", "cmeta", "print_replica_uuids", - "--fs_wal_dir=" + other_master->data_dir(), + "--fs_wal_dir=" + other_master->wal_dir(), "--fs_data_dirs=" + other_master->data_dir(), master::SysCatalogTable::kSysCatalogTabletId }; @@ -408,8 +407,8 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) { kBinPath, "fs", "format", - "--fs_wal_dir=" + data_root, - "--fs_data_dirs=" + data_root, + "--fs_wal_dir=" + failed_master->wal_dir(), + "--fs_data_dirs=" + failed_master->data_dir(), "--uuid=" + uuid }; ASSERT_OK(Subprocess::Call(args)); @@ -421,8 +420,8 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) { kBinPath, "local_replica", "copy_from_remote", - "--fs_wal_dir=" + data_root, - "--fs_data_dirs=" + data_root, + "--fs_wal_dir=" + failed_master->wal_dir(), + "--fs_data_dirs=" + failed_master->data_dir(), master::SysCatalogTable::kSysCatalogTabletId, other_master->bound_rpc_hostport().ToString() }; http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/master_migration-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/master_migration-itest.cc b/src/kudu/integration-tests/master_migration-itest.cc index d3fed08..29f2b2d 100644 --- a/src/kudu/integration-tests/master_migration-itest.cc +++ b/src/kudu/integration-tests/master_migration-itest.cc @@ -110,13 +110,15 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) { // Format a filesystem tree for each of the new masters and get the uuids. for (int i = 1; i < kMasterRpcPorts.size(); i++) { string data_root = cluster_->GetDataPath(Substitute("master-$0", i)); + string wal_dir = cluster_->GetWalPath(Substitute("master-$0", i)); ASSERT_OK(env_->CreateDir(DirName(data_root))); + ASSERT_OK(env_->CreateDir(wal_dir)); { vector<string> args = { kBinPath, "fs", "format", - "--fs_wal_dir=" + data_root, + "--fs_wal_dir=" + wal_dir, "--fs_data_dirs=" + data_root }; ASSERT_OK(Subprocess::Call(args)); @@ -127,7 +129,7 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) { "fs", "dump", "uuid", - "--fs_wal_dir=" + data_root, + "--fs_wal_dir=" + wal_dir, "--fs_data_dirs=" + data_root }; string uuid; @@ -145,7 +147,7 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) { "local_replica", "cmeta", "rewrite_raft_config", - "--fs_wal_dir=" + data_root, + "--fs_wal_dir=" + cluster_->GetWalPath("master-0"), "--fs_data_dirs=" + data_root, SysCatalogTable::kSysCatalogTabletId }; @@ -168,11 +170,12 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) { // filesystems. for (int i = 1; i < kMasterRpcPorts.size(); i++) { string data_root = cluster_->GetDataPath(Substitute("master-$0", i)); + string wal_dir = cluster_->GetWalPath(Substitute("master-$0", i)); vector<string> args = { kBinPath, "local_replica", "copy_from_remote", - "--fs_wal_dir=" + data_root, + "--fs_wal_dir=" + wal_dir, "--fs_data_dirs=" + data_root, SysCatalogTable::kSysCatalogTabletId, cluster_->master()->bound_rpc_hostport().ToString() http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/multidir_cluster-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/multidir_cluster-itest.cc b/src/kudu/integration-tests/multidir_cluster-itest.cc new file mode 100644 index 0000000..5b04ef4 --- /dev/null +++ b/src/kudu/integration-tests/multidir_cluster-itest.cc @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest.h> + +#include <map> +#include <string> +#include <vector> + +#include "kudu/client/client.h" +#include "kudu/gutil/map-util.h" +#include "kudu/integration-tests/external_mini_cluster-itest-base.h" +#include "kudu/integration-tests/test_workload.h" +#include "kudu/util/path_util.h" + +namespace kudu { + +using std::map; +using std::string; +using std::vector; + +class MultiDirClusterITest : public ExternalMiniClusterITestBase {}; + +TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) { + const uint32_t kNumDataDirs = 3; + vector<string> ts_flags = { + // Flush frequently to trigger writes. + "--flush_threshold_mb=1", + "--flush_threshold_secs=1", + + // Spread tablet data across all data dirs. + "--fs_target_data_dirs_per_tablet=0" + }; + + NO_FATALS(StartCluster(ts_flags, {}, /* num_tablet_servers= */ 1, kNumDataDirs)); + ExternalTabletServer* ts = cluster_->tablet_server(0); + TestWorkload work(cluster_.get()); + work.set_num_replicas(1); + work.Setup(); + + // Check that all daemons have the expected number of directories. + ASSERT_EQ(kNumDataDirs, cluster_->master()->data_dirs().size()); + ASSERT_EQ(kNumDataDirs, ts->data_dirs().size()); + + // Take an initial snapshot of the number of files in each directory. + map<string, int> num_files_in_each_dir; + for (const string& data_dir : ts->data_dirs()) { + string data_path = JoinPathSegments(data_dir, "data"); + vector<string> files; + ASSERT_OK(inspect_->ListFilesInDir(data_path, &files)); + InsertOrDie(&num_files_in_each_dir, data_dir, files.size()); + } + + work.Start(); + ASSERT_EVENTUALLY([&] { + // Check that files are being written to more than one directory. + int num_dirs_added_to = 0; + for (const string& data_dir : ts->data_dirs()) { + string data_path = JoinPathSegments(data_dir, "data"); + vector<string> files; + inspect_->ListFilesInDir(data_path, &files); + int* num_files_before_insert = FindOrNull(num_files_in_each_dir, data_dir); + ASSERT_NE(nullptr, num_files_before_insert); + if (*num_files_before_insert < files.size()) { + num_dirs_added_to++; + } + } + // Block placement should guarantee that more than one data dir will have + // data written to it. + ASSERT_GT(num_dirs_added_to, 1); + vector<string> wal_files; + ASSERT_OK(inspect_->ListFilesInDir(JoinPathSegments(ts->wal_dir(), "wals"), &wal_files)); + ASSERT_FALSE(wal_files.empty()); + }); + work.StopAndJoin(); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/open-readonly-fs-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/open-readonly-fs-itest.cc b/src/kudu/integration-tests/open-readonly-fs-itest.cc index 68e7ba0..e17902d 100644 --- a/src/kudu/integration-tests/open-readonly-fs-itest.cc +++ b/src/kudu/integration-tests/open-readonly-fs-itest.cc @@ -114,8 +114,8 @@ TEST_F(OpenReadonlyFsITest, TestWriteAndVerify) { auto t = std::thread([this, deadline] () { FsManagerOpts fs_opts; fs_opts.read_only = true; - fs_opts.wal_path = cluster_->tablet_server(0)->data_dir(); - fs_opts.data_paths = { cluster_->tablet_server(0)->data_dir() }; + fs_opts.wal_path = cluster_->tablet_server(0)->wal_dir(); + fs_opts.data_paths = cluster_->tablet_server(0)->data_dirs(); while (MonoTime::Now() < deadline) { FsManager fs(Env::Default(), fs_opts); CHECK_OK(fs.Open()); http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/ts_itest-base.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_itest-base.h b/src/kudu/integration-tests/ts_itest-base.h index 898a349..7c96bdd 100644 --- a/src/kudu/integration-tests/ts_itest-base.h +++ b/src/kudu/integration-tests/ts_itest-base.h @@ -86,7 +86,8 @@ class TabletServerIntegrationTestBase : public TabletServerTestBase { void CreateCluster(const std::string& data_root_path, const std::vector<std::string>& non_default_ts_flags, - const std::vector<std::string>& non_default_master_flags) { + const std::vector<std::string>& non_default_master_flags, + uint32_t num_data_dirs = 1) { LOG(INFO) << "Starting cluster with:"; LOG(INFO) << "--------------"; @@ -97,6 +98,7 @@ class TabletServerIntegrationTestBase : public TabletServerTestBase { ExternalMiniClusterOptions opts; opts.num_tablet_servers = FLAGS_num_tablet_servers; opts.data_root = GetTestPath(data_root_path); + opts.num_data_dirs = num_data_dirs; // Enable exactly once semantics for tests. http://git-wip-us.apache.org/repos/asf/kudu/blob/fc98b1ae/src/kudu/integration-tests/ts_recovery-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_recovery-itest.cc b/src/kudu/integration-tests/ts_recovery-itest.cc index a145cd6..e996ad1 100644 --- a/src/kudu/integration-tests/ts_recovery-itest.cc +++ b/src/kudu/integration-tests/ts_recovery-itest.cc @@ -322,7 +322,10 @@ TEST_F(TsRecoveryITestDeathTest, TestRecoverFromOpIdOverflow) { { // Append a no-op to the WAL with an overflowed term and index to simulate a // crash after KUDU-1933. - gscoped_ptr<FsManager> fs_manager(new FsManager(env_, ets->data_dir())); + FsManagerOpts opts; + opts.wal_path = ets->wal_dir(); + opts.data_paths = ets->data_dirs(); + gscoped_ptr<FsManager> fs_manager(new FsManager(env_, opts)); ASSERT_OK(fs_manager->Open()); scoped_refptr<Clock> clock(new HybridClock()); ASSERT_OK(clock->Init());
