internal_mini_cluster: support Cluster/LogVerifier This patch introduces a MiniCluster-agnostic MiniClusterFsInspector. With this, the LogVerifier, and thus, the ClusterVerifier can support both External- and InternalMiniClusters.
The LogVerifier would originally open a read-only FsManager (which in turn would open a BlockManager and a DataDirManager) per server and pass it to the LogReader to inspect the WALs. Instead, the LogVerifier now creates a MiniClusterFsInspector, which is more lightweight and defined per cluster. To the LogReader, it passes the WAL directory and an env, which is all the LogReader needed from the FsManager in the first place. To test, I updated a test case in ts_tablet_manager-itest to make use of the new ClusterVerifier with an internal cluster. CheckCluster() uses a LogVerifier, which in this case uses an MiniClusterFsInspector that operates on an InternalMiniCluster. Change-Id: I228a6e3ba1a42db4e243ffdc5116f0c60ee04a84 Reviewed-on: http://gerrit.cloudera.org:8080/9137 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2165ce57 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2165ce57 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2165ce57 Branch: refs/heads/master Commit: 2165ce577e208f27a18d3a98fdf97b9f3333b7d5 Parents: 270dd99 Author: Andrew Wong <[email protected]> Authored: Wed Jan 24 17:46:49 2018 -0800 Committer: Andrew Wong <[email protected]> Committed: Wed Feb 21 00:10:38 2018 +0000 ---------------------------------------------------------------------- src/kudu/consensus/log-test.cc | 2 +- src/kudu/consensus/log_reader.cc | 48 +-- src/kudu/consensus/log_reader.h | 26 +- src/kudu/fs/fs_manager.h | 4 +- src/kudu/integration-tests/CMakeLists.txt | 2 +- .../integration-tests/client_failover-itest.cc | 2 +- src/kudu/integration-tests/cluster_verifier.cc | 21 +- .../integration-tests/create-table-itest.cc | 2 +- .../integration-tests/delete_table-itest.cc | 2 +- .../external_mini_cluster-itest-base.cc | 4 +- .../external_mini_cluster-itest-base.h | 4 +- .../external_mini_cluster_fs_inspector.cc | 383 ------------------- .../external_mini_cluster_fs_inspector.h | 139 ------- src/kudu/integration-tests/log_verifier.cc | 65 ++-- src/kudu/integration-tests/log_verifier.h | 29 +- .../mini_cluster_fs_inspector.cc | 377 ++++++++++++++++++ .../mini_cluster_fs_inspector.h | 140 +++++++ .../raft_config_change-itest.cc | 2 +- .../raft_consensus-itest-base.cc | 2 +- .../integration-tests/raft_consensus-itest.cc | 5 +- .../raft_consensus_election-itest.cc | 2 +- .../raft_consensus_nonvoter-itest.cc | 2 +- src/kudu/integration-tests/tablet_copy-itest.cc | 2 +- .../tablet_copy_client_session-itest.cc | 2 +- .../tablet_replacement-itest.cc | 2 +- .../tombstoned_voting-itest.cc | 2 +- .../tombstoned_voting-stress-test.cc | 2 +- src/kudu/integration-tests/ts_itest-base.cc | 4 +- src/kudu/integration-tests/ts_itest-base.h | 4 +- src/kudu/integration-tests/ts_recovery-itest.cc | 8 +- .../ts_tablet_manager-itest.cc | 28 +- src/kudu/mini-cluster/external_mini_cluster.cc | 16 +- src/kudu/mini-cluster/external_mini_cluster.h | 10 + src/kudu/mini-cluster/internal_mini_cluster.cc | 9 + src/kudu/mini-cluster/internal_mini_cluster.h | 11 + src/kudu/mini-cluster/mini_cluster.h | 10 + src/kudu/tablet/tablet_bootstrap.cc | 19 +- src/kudu/tools/kudu-tool-test.cc | 8 +- src/kudu/tools/kudu-ts-cli-test.cc | 2 +- 39 files changed, 705 insertions(+), 697 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/consensus/log-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/consensus/log-test.cc b/src/kudu/consensus/log-test.cc index 11aca81..3d56b96 100644 --- a/src/kudu/consensus/log-test.cc +++ b/src/kudu/consensus/log-test.cc @@ -778,7 +778,7 @@ TEST_P(LogTestOptionalCompression, TestWriteManyBatches) { // seg003: 0.20 through 0.29 // seg004: 0.30 through 0.39 TEST_P(LogTestOptionalCompression, TestLogReader) { - LogReader reader(fs_manager_.get(), + LogReader reader(env_, scoped_refptr<LogIndex>(), kTestTablet, nullptr); http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/consensus/log_reader.cc ---------------------------------------------------------------------- diff --git a/src/kudu/consensus/log_reader.cc b/src/kudu/consensus/log_reader.cc index 88adbb5..d357f23 100644 --- a/src/kudu/consensus/log_reader.cc +++ b/src/kudu/consensus/log_reader.cc @@ -74,44 +74,36 @@ struct LogSegmentSeqnoComparator { const int64_t LogReader::kNoSizeLimit = -1; -Status LogReader::Open(FsManager* fs_manager, +Status LogReader::Open(Env* env, + const string& tablet_wal_dir, const scoped_refptr<LogIndex>& index, const string& tablet_id, const scoped_refptr<MetricEntity>& metric_entity, shared_ptr<LogReader>* reader) { auto log_reader = std::make_shared<LogReader>( - fs_manager, index, tablet_id, metric_entity); - - string tablet_wal_path = fs_manager->GetTabletWalDir(tablet_id); + env, index, tablet_id, metric_entity); - RETURN_NOT_OK(log_reader->Init(tablet_wal_path)) + RETURN_NOT_OK_PREPEND(log_reader->Init(tablet_wal_dir), + "Unable to initialize log reader") *reader = log_reader; return Status::OK(); } -Status LogReader::OpenFromRecoveryDir(FsManager* fs_manager, - const string& tablet_id, - const scoped_refptr<MetricEntity>& metric_entity, - shared_ptr<LogReader>* reader) { - string recovery_path = fs_manager->GetTabletWalRecoveryDir(tablet_id); - - // When recovering, we don't want to have any log index -- since it isn't fsynced() - // during writing, its contents are useless to us. - scoped_refptr<LogIndex> index(nullptr); - auto log_reader = std::make_shared<LogReader>( - fs_manager, index, tablet_id, metric_entity); - RETURN_NOT_OK_PREPEND(log_reader->Init(recovery_path), - "Unable to initialize log reader"); - *reader = log_reader; - return Status::OK(); +Status LogReader::Open(FsManager* fs_manager, + const scoped_refptr<LogIndex>& index, + const std::string& tablet_id, + const scoped_refptr<MetricEntity>& metric_entity, + std::shared_ptr<LogReader>* reader) { + return LogReader::Open(fs_manager->env(), fs_manager->GetTabletWalDir(tablet_id), + index, tablet_id, metric_entity, reader); } -LogReader::LogReader(FsManager* fs_manager, - const scoped_refptr<LogIndex>& index, +LogReader::LogReader(Env* env, + scoped_refptr<LogIndex> index, string tablet_id, const scoped_refptr<MetricEntity>& metric_entity) - : fs_manager_(fs_manager), - log_index_(index), + : env_(env), + log_index_(std::move(index)), tablet_id_(std::move(tablet_id)), state_(kLogReaderInitialized) { if (metric_entity) { @@ -131,9 +123,7 @@ Status LogReader::Init(const string& tablet_wal_path) { } VLOG(1) << "Reading wal from path:" << tablet_wal_path; - Env* env = fs_manager_->env(); - - if (!fs_manager_->Exists(tablet_wal_path)) { + if (!env_->FileExists(tablet_wal_path)) { return Status::IllegalState("Cannot find wal location at", tablet_wal_path); } @@ -141,7 +131,7 @@ Status LogReader::Init(const string& tablet_wal_path) { // list existing segment files vector<string> log_files; - RETURN_NOT_OK_PREPEND(env->GetChildren(tablet_wal_path, &log_files), + RETURN_NOT_OK_PREPEND(env_->GetChildren(tablet_wal_path, &log_files), "Unable to read children from path"); SegmentSequence read_segments; @@ -151,7 +141,7 @@ Status LogReader::Init(const string& tablet_wal_path) { if (HasPrefixString(log_file, FsManager::kWalFileNamePrefix)) { string fqp = JoinPathSegments(tablet_wal_path, log_file); scoped_refptr<ReadableLogSegment> segment; - Status s = ReadableLogSegment::Open(env, fqp, &segment); + Status s = ReadableLogSegment::Open(env_, fqp, &segment); if (s.IsUninitialized()) { // This indicates that the segment was created but the writer // crashed before the header was successfully written. In this http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/consensus/log_reader.h ---------------------------------------------------------------------- diff --git a/src/kudu/consensus/log_reader.h b/src/kudu/consensus/log_reader.h index 312868f..bc88cb3 100644 --- a/src/kudu/consensus/log_reader.h +++ b/src/kudu/consensus/log_reader.h @@ -35,6 +35,7 @@ namespace kudu { class Counter; +class Env; class FsManager; class Histogram; class MetricEntity; @@ -57,23 +58,25 @@ class LogReader { public: ~LogReader(); - // Opens a LogReader on the default tablet log directory, and sets - // 'reader' to the newly created LogReader. + // Opens a LogReader on the tablet log directory specified by + // 'tablet_wal_dir', and sets 'reader' to the newly created LogReader. // // 'index' may be NULL, but if it is, ReadReplicatesInRange() may not // be used. - static Status Open(FsManager* fs_manager, + static Status Open(Env* env, + const std::string& tablet_wal_dir, const scoped_refptr<LogIndex>& index, const std::string& tablet_id, const scoped_refptr<MetricEntity>& metric_entity, std::shared_ptr<LogReader>* reader); - // Opens a LogReader on a specific tablet log recovery directory, and sets - // 'reader' to the newly created LogReader. - static Status OpenFromRecoveryDir(FsManager* fs_manager, - const std::string& tablet_id, - const scoped_refptr<MetricEntity>& metric_entity, - std::shared_ptr<LogReader>* reader); + // Same as above, but will use `fs_manager` to determine the default WAL dir + // for the tablet. + static Status Open(FsManager* fs_manager, + const scoped_refptr<LogIndex>& index, + const std::string& tablet_id, + const scoped_refptr<MetricEntity>& metric_entity, + std::shared_ptr<LogReader>* reader); // Return the minimum replicate index that is retained in the currently available // logs. May return -1 if no replicates have been logged. @@ -165,8 +168,7 @@ class LogReader { faststring* tmp_buf, gscoped_ptr<LogEntryBatchPB>* batch) const; - LogReader(FsManager* fs_manager, const scoped_refptr<LogIndex>& index, - std::string tablet_id, + LogReader(Env* env, scoped_refptr<LogIndex> index, std::string tablet_id, const scoped_refptr<MetricEntity>& metric_entity); // Reads the headers of all segments in 'tablet_wal_path'. @@ -175,7 +177,7 @@ class LogReader { // Initializes an 'empty' reader for tests, i.e. does not scan a path looking for segments. Status InitEmptyReaderForTests(); - FsManager *fs_manager_; + Env* env_; const scoped_refptr<LogIndex> log_index_; const std::string tablet_id_; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/fs/fs_manager.h ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h index 955d159..9a29e57 100644 --- a/src/kudu/fs/fs_manager.h +++ b/src/kudu/fs/fs_manager.h @@ -56,7 +56,7 @@ struct CreateBlockOptions; } // namespace fs namespace itest { -class ExternalMiniClusterFsInspector; +class MiniClusterFsInspector; } // namespace itest namespace tserver { @@ -264,7 +264,7 @@ class FsManager { FRIEND_TEST(FsManagerTestBase, TestMetadataDirInDataRoot); FRIEND_TEST(FsManagerTestBase, TestIsolatedMetadataDir); FRIEND_TEST(tserver::MiniTabletServerTest, TestFsLayoutEndToEnd); - friend class itest::ExternalMiniClusterFsInspector; // for access to directory names + friend class itest::MiniClusterFsInspector; // for access to directory names // Initializes, sanitizes, and canonicalizes the filesystem roots. // Determines the correct filesystem root for tablet-specific metadata. http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 17fbc68..af5dae9 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -24,9 +24,9 @@ set(INTEGRATION_TESTS_SRCS cluster_itest_util.cc cluster_verifier.cc external_mini_cluster-itest-base.cc - external_mini_cluster_fs_inspector.cc internal_mini_cluster-itest-base.cc log_verifier.cc + mini_cluster_fs_inspector.cc raft_consensus-itest-base.cc test_workload.cc ts_itest-base.cc http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/client_failover-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/client_failover-itest.cc b/src/kudu/integration-tests/client_failover-itest.cc index 47cedf4..fb23592 100644 --- a/src/kudu/integration-tests/client_failover-itest.cc +++ b/src/kudu/integration-tests/client_failover-itest.cc @@ -36,7 +36,7 @@ #include "kudu/gutil/map-util.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tablet/metadata.pb.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/cluster_verifier.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/cluster_verifier.cc b/src/kudu/integration-tests/cluster_verifier.cc index 4e2147b..b903a81 100644 --- a/src/kudu/integration-tests/cluster_verifier.cc +++ b/src/kudu/integration-tests/cluster_verifier.cc @@ -29,7 +29,6 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/log_verifier.h" -#include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/mini-cluster/mini_cluster.h" #include "kudu/tools/ksck.h" #include "kudu/tools/ksck_remote.h" @@ -44,7 +43,6 @@ using std::vector; namespace kudu { -using cluster::ExternalMiniCluster; using cluster::MiniCluster; using strings::Substitute; using tools::Ksck; @@ -89,17 +87,14 @@ void ClusterVerifier::CheckCluster() { } ASSERT_OK(s); - // TODO(todd): we should support LogVerifier on internal clusters! - if (ExternalMiniCluster* emc = dynamic_cast<ExternalMiniCluster*>(cluster_)) { - // Verify that the committed op indexes match up across the servers. - // We have to use "AssertEventually" here because many tests verify clusters - // while they are still running, and the verification can fail spuriously in - // the case that - LogVerifier lv(emc); - AssertEventually([&]() { - ASSERT_OK(lv.VerifyCommittedOpIdsMatch()); - }); - } + LogVerifier lv(cluster_); + // Verify that the committed op indexes match up across the servers. We have + // to use "AssertEventually" here because many tests verify clusters while + // they are still running, and the verification can fail spuriously in the + // case that + AssertEventually([&]() { + ASSERT_OK(lv.VerifyCommittedOpIdsMatch()); + }); } Status ClusterVerifier::DoKsck() { http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/create-table-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/create-table-itest.cc b/src/kudu/integration-tests/create-table-itest.cc index b223156..1bed8ff 100644 --- a/src/kudu/integration-tests/create-table-itest.cc +++ b/src/kudu/integration-tests/create-table-itest.cc @@ -43,7 +43,7 @@ #include "kudu/gutil/ref_counted.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/delete_table-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/delete_table-itest.cc b/src/kudu/integration-tests/delete_table-itest.cc index 3424e58..12b6848 100644 --- a/src/kudu/integration-tests/delete_table-itest.cc +++ b/src/kudu/integration-tests/delete_table-itest.cc @@ -54,7 +54,7 @@ #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/external_mini_cluster-itest-base.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc index db7d83b..756a1b9 100644 --- a/src/kudu/integration-tests/external_mini_cluster-itest-base.cc +++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.cc @@ -28,7 +28,7 @@ #include "kudu/gutil/stl_util.h" #include "kudu/integration-tests/cluster_itest_util.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/util/pstack_watcher.h" #include "kudu/util/status.h" @@ -64,7 +64,7 @@ void ExternalMiniClusterITestBase::StartClusterWithOpts( ExternalMiniClusterOptions opts) { cluster_.reset(new ExternalMiniCluster(std::move(opts))); ASSERT_OK(cluster_->Start()); - inspect_.reset(new itest::ExternalMiniClusterFsInspector(cluster_.get())); + inspect_.reset(new itest::MiniClusterFsInspector(cluster_.get())); ASSERT_OK(itest::CreateTabletServerMap(cluster_->master_proxy(), cluster_->messenger(), &ts_map_)); http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/external_mini_cluster-itest-base.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster-itest-base.h b/src/kudu/integration-tests/external_mini_cluster-itest-base.h index eba956c..29c2c57 100644 --- a/src/kudu/integration-tests/external_mini_cluster-itest-base.h +++ b/src/kudu/integration-tests/external_mini_cluster-itest-base.h @@ -23,7 +23,7 @@ #include <vector> #include "kudu/client/shared_ptr.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/util/test_util.h" @@ -53,7 +53,7 @@ class ExternalMiniClusterITestBase : public KuduTest { void StopCluster(); std::unique_ptr<cluster::ExternalMiniCluster> cluster_; - std::unique_ptr<itest::ExternalMiniClusterFsInspector> inspect_; + std::unique_ptr<itest::MiniClusterFsInspector> inspect_; client::sp::shared_ptr<client::KuduClient> client_; std::unordered_map<std::string, itest::TServerDetails*> ts_map_; }; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc deleted file mode 100644 index 911f9ea..0000000 --- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc +++ /dev/null @@ -1,383 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" - -#include <algorithm> -#include <set> - -#include <glog/logging.h> - -#include "kudu/consensus/metadata.pb.h" -#include "kudu/gutil/strings/join.h" -#include "kudu/gutil/strings/substitute.h" -#include "kudu/gutil/strings/util.h" -#include "kudu/fs/fs_manager.h" -#include "kudu/mini-cluster/external_mini_cluster.h" -#include "kudu/util/env.h" -#include "kudu/util/env_util.h" -#include "kudu/util/monotime.h" -#include "kudu/util/path_util.h" -#include "kudu/util/pb_util.h" -#include "kudu/util/status.h" - -namespace kudu { -namespace itest { - -using std::set; -using std::string; -using std::vector; -using cluster::ExternalMiniCluster; -using consensus::ConsensusMetadataPB; -using env_util::ListFilesInDir; -using strings::Substitute; -using tablet::TabletDataState; -using tablet::TabletSuperBlockPB; - -ExternalMiniClusterFsInspector::ExternalMiniClusterFsInspector(ExternalMiniCluster* cluster) - : env_(Env::Default()), - cluster_(CHECK_NOTNULL(cluster)) { -} - -ExternalMiniClusterFsInspector::~ExternalMiniClusterFsInspector() {} - -int ExternalMiniClusterFsInspector::CountFilesInDir(const string& path, - StringPiece pattern) { - vector<string> entries; - Status s = ListFilesInDir(env_, path, &entries); - if (!s.ok()) return 0; - return std::count_if(entries.begin(), entries.end(), [&](const string& s) { - return pattern.empty() || MatchPattern(s, pattern); - }); -} - -int ExternalMiniClusterFsInspector::CountWALFilesOnTS(int index) { - string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), - FsManager::kWalDirName); - vector<string> tablets; - CHECK_OK(ListFilesInDir(env_, ts_wal_dir, &tablets)); - int total_segments = 0; - for (const string& tablet : tablets) { - string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet); - total_segments += CountFilesInDir(tablet_wal_dir); - } - return total_segments; -} - -vector<string> ExternalMiniClusterFsInspector::ListTablets() { - set<string> tablets; - for (int i = 0; i < cluster_->num_tablet_servers(); i++) { - auto ts_tablets = ListTabletsOnTS(i); - tablets.insert(ts_tablets.begin(), ts_tablets.end()); - } - return vector<string>(tablets.begin(), tablets.end()); -} - -vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) { - string wal_dir = cluster_->tablet_server(index)->wal_dir(); - string meta_dir = JoinPathSegments(wal_dir, FsManager::kTabletMetadataDirName); - vector<string> tablets; - CHECK_OK(ListFilesInDir(env_, meta_dir, &tablets)); - return tablets; -} - -vector<string> ExternalMiniClusterFsInspector::ListTabletsWithDataOnTS(int index) { - string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), - FsManager::kWalDirName); - vector<string> tablets; - CHECK_OK(ListFilesInDir(env_, wal_dir, &tablets)); - return tablets; -} - -int ExternalMiniClusterFsInspector::CountFilesInWALDirForTS( - int index, - const string& tablet_id, - StringPiece pattern) { - string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), - FsManager::kWalDirName); - string tablet_wal_dir = JoinPathSegments(wal_dir, tablet_id); - if (!env_->FileExists(tablet_wal_dir)) { - return 0; - } - return CountFilesInDir(tablet_wal_dir, pattern); -} - -bool ExternalMiniClusterFsInspector::DoesConsensusMetaExistForTabletOnTS(int index, - const string& tablet_id) { - ConsensusMetadataPB cmeta_pb; - Status s = ReadConsensusMetadataOnTS(index, tablet_id, &cmeta_pb); - return s.ok(); -} - -int ExternalMiniClusterFsInspector::CountReplicasInMetadataDirs() { - // Rather than using FsManager's functionality for listing blocks, we just manually - // list the contents of the metadata directory. This is because we're using an - // external minicluster, and initializing a new FsManager to point at the running - // tablet servers isn't easy. - int count = 0; - for (int i = 0; i < cluster_->num_tablet_servers(); i++) { - string wal_dir = cluster_->tablet_server(i)->wal_dir(); - count += CountFilesInDir(JoinPathSegments(wal_dir, FsManager::kTabletMetadataDirName)); - } - return count; -} - -Status ExternalMiniClusterFsInspector::CheckNoDataOnTS(int index) { - const string& wal_dir = cluster_->tablet_server(index)->wal_dir(); - if (CountFilesInDir(JoinPathSegments(wal_dir, FsManager::kTabletMetadataDirName)) > 0) { - return Status::IllegalState("tablet metadata blocks still exist", wal_dir); - } - if (CountWALFilesOnTS(index) > 0) { - return Status::IllegalState("wals still exist", wal_dir); - } - if (CountFilesInDir(JoinPathSegments(wal_dir, FsManager::kConsensusMetadataDirName)) > 0) { - return Status::IllegalState("consensus metadata still exists", wal_dir); - } - return Status::OK();; -} - -Status ExternalMiniClusterFsInspector::CheckNoData() { - for (int i = 0; i < cluster_->num_tablet_servers(); i++) { - RETURN_NOT_OK(CheckNoDataOnTS(i)); - } - return Status::OK();; -} - -string ExternalMiniClusterFsInspector::GetTabletSuperBlockPathOnTS(int ts_index, - const string& tablet_id) const { - string wal_dir = cluster_->tablet_server(ts_index)->wal_dir(); - string meta_dir = JoinPathSegments(wal_dir, FsManager::kTabletMetadataDirName); - return JoinPathSegments(meta_dir, tablet_id); -} - -Status ExternalMiniClusterFsInspector::ReadTabletSuperBlockOnTS(int index, - const string& tablet_id, - TabletSuperBlockPB* sb) { - const auto& sb_path = GetTabletSuperBlockPathOnTS(index, tablet_id); - return pb_util::ReadPBContainerFromPath(env_, sb_path, sb); -} - -int64_t ExternalMiniClusterFsInspector::GetTabletSuperBlockMTimeOrDie(int ts_index, - const string& tablet_id) { - int64_t timestamp; - CHECK_OK(env_->GetFileModifiedTime(GetTabletSuperBlockPathOnTS(ts_index, tablet_id), ×tamp)); - return timestamp; -} - -string ExternalMiniClusterFsInspector::GetConsensusMetadataPathOnTS(int index, - const string& tablet_id) const { - string wal_dir = cluster_->tablet_server(index)->wal_dir(); - string cmeta_dir = JoinPathSegments(wal_dir, FsManager::kConsensusMetadataDirName); - return JoinPathSegments(cmeta_dir, tablet_id); -} - -Status ExternalMiniClusterFsInspector::ReadConsensusMetadataOnTS(int index, - const string& tablet_id, - ConsensusMetadataPB* cmeta_pb) { - auto cmeta_path = GetConsensusMetadataPathOnTS(index, tablet_id); - if (!env_->FileExists(cmeta_path)) { - return Status::NotFound("Consensus metadata file not found", cmeta_path); - } - return pb_util::ReadPBContainerFromPath(env_, cmeta_path, cmeta_pb); -} - -Status ExternalMiniClusterFsInspector::WriteConsensusMetadataOnTS( - int index, - const string& tablet_id, - const ConsensusMetadataPB& cmeta_pb) { - auto cmeta_path = GetConsensusMetadataPathOnTS(index, tablet_id); - return pb_util::WritePBContainerToPath(env_, cmeta_path, cmeta_pb, - pb_util::OVERWRITE, pb_util::NO_SYNC); -} - - -Status ExternalMiniClusterFsInspector::CheckTabletDataStateOnTS( - int index, - const string& tablet_id, - const vector<TabletDataState>& allowed_states) { - - TabletSuperBlockPB sb; - RETURN_NOT_OK(ReadTabletSuperBlockOnTS(index, tablet_id, &sb)); - if (std::find(allowed_states.begin(), allowed_states.end(), sb.tablet_data_state()) != - allowed_states.end()) { - return Status::OK(); - } - - vector<string> state_names; - for (auto state : allowed_states) { - state_names.push_back(TabletDataState_Name(state)); - } - string expected_str = JoinStrings(state_names, ","); - if (state_names.size() > 1) { - expected_str = "one of: " + expected_str; - } - - return Status::IllegalState(Substitute("State $0 unexpected, expected $1", - TabletDataState_Name(sb.tablet_data_state()), - expected_str)); -} - -Status ExternalMiniClusterFsInspector::WaitForNoData(const MonoDelta& timeout) { - MonoTime deadline = MonoTime::Now() + timeout; - Status s; - while (true) { - s = CheckNoData(); - if (s.ok()) return Status::OK(); - if (deadline < MonoTime::Now()) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(10)); - } - return Status::TimedOut("Timed out waiting for no data", s.ToString()); -} - -Status ExternalMiniClusterFsInspector::WaitForNoDataOnTS(int index, const MonoDelta& timeout) { - MonoTime deadline = MonoTime::Now() + timeout; - Status s; - while (true) { - s = CheckNoDataOnTS(index); - if (s.ok()) return Status::OK(); - if (deadline < MonoTime::Now()) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(10)); - } - return Status::TimedOut("Timed out waiting for no data", s.ToString()); -} - -Status ExternalMiniClusterFsInspector::WaitForMinFilesInTabletWalDirOnTS(int index, - const string& tablet_id, - int count, - const MonoDelta& timeout) { - int seen = 0; - MonoTime deadline = MonoTime::Now() + timeout; - while (true) { - seen = CountFilesInWALDirForTS(index, tablet_id); - if (seen >= count) return Status::OK(); - if (deadline < MonoTime::Now()) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(10)); - } - return Status::TimedOut(Substitute("Timed out waiting for number of WAL segments on tablet $0 " - "on TS $1 to be $2. Found $3", - tablet_id, index, count, seen)); -} - -Status ExternalMiniClusterFsInspector::WaitForReplicaCount(int expected, const MonoDelta& timeout) { - const MonoTime deadline = MonoTime::Now() + timeout; - int found; - while (true) { - found = CountReplicasInMetadataDirs(); - if (found == expected) { - return Status::OK(); - } - if (MonoTime::Now() > deadline) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(10)); - } - return Status::TimedOut( - Substitute("Timed out waiting for a total replica count of $0. " - "Found $1 replicas", expected, found)); -} - -Status ExternalMiniClusterFsInspector::WaitForTabletDataStateOnTS( - int index, - const string& tablet_id, - const vector<TabletDataState>& expected_states, - const MonoDelta& timeout) { - MonoTime start = MonoTime::Now(); - MonoTime deadline = start + timeout; - Status s; - while (true) { - s = CheckTabletDataStateOnTS(index, tablet_id, expected_states); - if (s.ok()) return Status::OK(); - if (MonoTime::Now() > deadline) break; - SleepFor(MonoDelta::FromMilliseconds(5)); - } - return Status::TimedOut(Substitute("Timed out after $0 waiting for correct tablet state: $1", - (MonoTime::Now() - start).ToString(), - s.ToString())); -} - -Status ExternalMiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs( - int ts_index, const string& tablet_id, - const vector<string>& substrings_required, - const vector<string>& substrings_disallowed, - const MonoDelta& timeout) { - Status s; - MonoTime deadline = MonoTime::Now() + timeout; - - string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(ts_index)->wal_dir(), - FsManager::kWalDirName); - string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet_id); - - string error_msg; - vector<string> entries; - while (true) { - Status s = ListFilesInDir(env_, tablet_wal_dir, &entries); - std::sort(entries.begin(), entries.end()); - - error_msg = ""; - bool any_missing_required = false; - for (const string& required_filter : substrings_required) { - bool filter_matched = false; - for (const string& entry : entries) { - if (entry.find(required_filter) != string::npos) { - filter_matched = true; - break; - } - } - if (!filter_matched) { - any_missing_required = true; - error_msg += "missing from substrings_required: " + required_filter + "; "; - break; - } - } - - bool any_present_disallowed = false; - for (const string& entry : entries) { - if (any_present_disallowed) break; - for (const string& disallowed_filter : substrings_disallowed) { - if (entry.find(disallowed_filter) != string::npos) { - any_present_disallowed = true; - error_msg += "present from substrings_disallowed: " + entry + - " (" + disallowed_filter + "); "; - break; - } - } - } - - if (!any_missing_required && !any_present_disallowed) { - return Status::OK(); - } - if (MonoTime::Now() > deadline) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(10)); - } - - return Status::TimedOut(Substitute("Timed out waiting for file pattern on " - "tablet $0 on TS $1 in directory $2", - tablet_id, ts_index, tablet_wal_dir), - error_msg + "entries: " + JoinStrings(entries, ", ")); -} - -} // namespace itest -} // namespace kudu - http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h deleted file mode 100644 index fd82a89..0000000 --- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h +++ /dev/null @@ -1,139 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#pragma once - -#include <cstdint> -#include <string> -#include <vector> - -#include "kudu/gutil/macros.h" -#include "kudu/gutil/strings/stringpiece.h" -#include "kudu/tablet/metadata.pb.h" -#include "kudu/util/monotime.h" - -namespace kudu { - -class Env; -class Status; - -namespace cluster { -class ExternalMiniCluster; -} // namespace cluster - -namespace consensus { -class ConsensusMetadataPB; -} // namespace consensus - -namespace itest { - -// Utility class that digs around in a tablet server's data directory and -// provides methods useful for integration testing. This class must outlive -// the Env and ExternalMiniCluster objects that are passed into it. -class ExternalMiniClusterFsInspector { - public: - // Does not take ownership of the ExternalMiniCluster pointer. - explicit ExternalMiniClusterFsInspector(cluster::ExternalMiniCluster* cluster); - ~ExternalMiniClusterFsInspector(); - - // If provided, files are filtered by the glob-style pattern 'pattern'. - int CountFilesInDir(const std::string& path, StringPiece pattern = StringPiece()); - - // List all of the tablets with tablet metadata in the cluster. - std::vector<std::string> ListTablets(); - - // List all of the tablets with tablet metadata on the given tablet server index. - // This may include tablets that are tombstoned and not running. - std::vector<std::string> ListTabletsOnTS(int index); - - // List the tablet IDs on the given tablet which actually have data (as - // evidenced by their having a WAL). This excludes those that are tombstoned. - std::vector<std::string> ListTabletsWithDataOnTS(int index); - - // Return the number of files in the WAL directory for the given 'tablet_id' on TS 'index'. - // If provided, files are filtered by the glob-style pattern 'pattern'. - int CountFilesInWALDirForTS(int index, - const std::string& tablet_id, - StringPiece pattern = StringPiece()); - - bool DoesConsensusMetaExistForTabletOnTS(int index, const std::string& tablet_id); - - int CountReplicasInMetadataDirs(); - Status CheckNoDataOnTS(int index); - Status CheckNoData(); - - Status ReadTabletSuperBlockOnTS(int index, const std::string& tablet_id, - tablet::TabletSuperBlockPB* sb); - - // Get the modification time (in micros) of the tablet superblock for the given tablet - // server index and tablet ID. - int64_t GetTabletSuperBlockMTimeOrDie(int ts_index, const std::string& tablet_id); - - Status ReadConsensusMetadataOnTS(int index, const std::string& tablet_id, - consensus::ConsensusMetadataPB* cmeta_pb); - Status WriteConsensusMetadataOnTS(int index, - const std::string& tablet_id, - const consensus::ConsensusMetadataPB& cmeta_pb); - - Status CheckTabletDataStateOnTS(int index, - const std::string& tablet_id, - const std::vector<tablet::TabletDataState>& expected_states); - - Status WaitForNoData(const MonoDelta& timeout = MonoDelta::FromSeconds(30)); - Status WaitForNoDataOnTS(int index, const MonoDelta& timeout = MonoDelta::FromSeconds(30)); - Status WaitForMinFilesInTabletWalDirOnTS(int index, - const std::string& tablet_id, - int count, - const MonoDelta& timeout = MonoDelta::FromSeconds(60)); - Status WaitForReplicaCount(int expected, const MonoDelta& timeout = MonoDelta::FromSeconds(30)); - Status WaitForTabletDataStateOnTS(int index, - const std::string& tablet_id, - const std::vector<tablet::TabletDataState>& expected_states, - const MonoDelta& timeout = MonoDelta::FromSeconds(30)); - - // Loop and check for certain filenames in the WAL directory of the specified - // tablet. This function returns OK if we reach a state where: - // * For each string in 'substrings_required', we find *at least one file* - // whose name contains that string, and: - // * For each string in 'substrings_disallowed', we find *no files* whose name - // contains that string, even if the file also matches a string in the - // 'substrings_required'. - Status WaitForFilePatternInTabletWalDirOnTs( - int ts_index, - const std::string& tablet_id, - const std::vector<std::string>& substrings_required, - const std::vector<std::string>& substrings_disallowed, - const MonoDelta& timeout = MonoDelta::FromSeconds(30)); - - private: - // Return the number of files in WAL directories on the given tablet server. - // This includes log index files (not just segments). - int CountWALFilesOnTS(int index); - - std::string GetConsensusMetadataPathOnTS(int index, - const std::string& tablet_id) const; - - std::string GetTabletSuperBlockPathOnTS(int ts_index, - const std::string& tablet_id) const; - - Env* const env_; - cluster::ExternalMiniCluster* const cluster_; - - DISALLOW_COPY_AND_ASSIGN(ExternalMiniClusterFsInspector); -}; - -} // namespace itest -} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/log_verifier.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/log_verifier.cc b/src/kudu/integration-tests/log_verifier.cc index 3a04350..beeb546 100644 --- a/src/kudu/integration-tests/log_verifier.cc +++ b/src/kudu/integration-tests/log_verifier.cc @@ -36,14 +36,14 @@ #include "kudu/consensus/log_reader.h" #include "kudu/consensus/log_util.h" #include "kudu/consensus/opid.pb.h" -#include "kudu/fs/fs_manager.h" #include "kudu/gutil/map-util.h" #include "kudu/gutil/ref_counted.h" #include "kudu/gutil/strings/substitute.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" -#include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" +#include "kudu/mini-cluster/mini_cluster.h" #include "kudu/util/env.h" #include "kudu/util/metrics.h" +#include "kudu/util/path_util.h" #include "kudu/util/status.h" using std::map; @@ -56,39 +56,26 @@ using strings::Substitute; namespace kudu { -using cluster::ExternalMiniCluster; -using cluster::ExternalTabletServer; +using cluster::MiniCluster; using consensus::OpId; +using itest::MiniClusterFsInspector; using log::LogReader; -using itest::ExternalMiniClusterFsInspector; -LogVerifier::LogVerifier(ExternalMiniCluster* cluster) - : cluster_(cluster) { +LogVerifier::LogVerifier(cluster::MiniCluster* cluster) + : cluster_(cluster), + env_(cluster->env()) { + inspector_.reset(new MiniClusterFsInspector(cluster)); + CHECK(inspector_); } -LogVerifier::~LogVerifier() { -} - -Status LogVerifier::OpenFsManager(ExternalTabletServer* ets, - unique_ptr<FsManager>* fs) { - FsManagerOpts fs_opts; - fs_opts.read_only = true; - fs_opts.wal_root = ets->wal_dir(); - fs_opts.data_roots = ets->data_dirs(); - fs_opts.block_manager_type = cluster_->block_manager_type(); - - unique_ptr<FsManager> ret(new FsManager(Env::Default(), fs_opts)); - RETURN_NOT_OK_PREPEND(ret->Open(), - Substitute("Couldn't initialize FS Manager for $0", ets->wal_dir())); - fs->swap(ret); - return Status::OK(); -} +LogVerifier::~LogVerifier() {} -Status LogVerifier::ScanForCommittedOpIds(FsManager* fs, const string& tablet_id, +Status LogVerifier::ScanForCommittedOpIds(int ts_idx, const string& tablet_id, map<int64_t, int64_t>* index_to_term) { shared_ptr<LogReader> reader; - RETURN_NOT_OK(LogReader::Open(fs, scoped_refptr<log::LogIndex>(), tablet_id, + const string wal_dir = JoinPathSegments(inspector_->WalDirForTS(ts_idx), tablet_id); + RETURN_NOT_OK(LogReader::Open(env_, wal_dir, scoped_refptr<log::LogIndex>(), tablet_id, scoped_refptr<MetricEntity>(), &reader)); log::SegmentSequence segs; RETURN_NOT_OK(reader->GetSegmentsSnapshot(&segs)); @@ -113,14 +100,13 @@ Status LogVerifier::ScanForCommittedOpIds(FsManager* fs, const string& tablet_id return Status::OK(); } -Status LogVerifier::ScanForHighestCommittedOpIdInLog(ExternalTabletServer* ets, +Status LogVerifier::ScanForHighestCommittedOpIdInLog(int ts_idx, const string& tablet_id, OpId* commit_id) { - unique_ptr<FsManager> fs; - RETURN_NOT_OK(OpenFsManager(ets, &fs)); - const string& wal_dir = fs->GetTabletWalDir(tablet_id); + const string& wal_dir = inspector_->WalDirForTS(ts_idx); map<int64_t, int64_t> index_to_term; - RETURN_NOT_OK_PREPEND(ScanForCommittedOpIds(fs.get(), tablet_id, &index_to_term), + + RETURN_NOT_OK_PREPEND(ScanForCommittedOpIds(ts_idx, tablet_id, &index_to_term), Substitute("Couldn't scan log in dir $0", wal_dir)); if (index_to_term.empty()) { return Status::NotFound("no COMMITs in log"); @@ -131,10 +117,7 @@ Status LogVerifier::ScanForHighestCommittedOpIdInLog(ExternalTabletServer* ets, } Status LogVerifier::VerifyCommittedOpIdsMatch() { - ExternalMiniClusterFsInspector inspect(cluster_); - Env* env = Env::Default(); - - for (const string& tablet_id : inspect.ListTablets()) { + for (const string& tablet_id : inspector_->ListTablets()) { LOG(INFO) << "Checking tablet " << tablet_id; // Union set of the op indexes seen on any server. @@ -145,12 +128,10 @@ Status LogVerifier::VerifyCommittedOpIdsMatch() { // Gather the [index->term] map for each of the tablet servers // hosting this tablet. for (int i = 0; i < cluster_->num_tablet_servers(); i++) { - unique_ptr<FsManager> fs; - RETURN_NOT_OK(OpenFsManager(cluster_->tablet_server(i), &fs)); - const string& wal_dir = fs->GetTabletWalDir(tablet_id); - if (!env->FileExists(wal_dir)) continue; + const string& wal_dir = JoinPathSegments(inspector_->WalDirForTS(i), tablet_id); + if (!env_->FileExists(wal_dir)) continue; map<int64_t, int64_t> index_to_term; - RETURN_NOT_OK_PREPEND(ScanForCommittedOpIds(fs.get(), tablet_id, &index_to_term), + RETURN_NOT_OK_PREPEND(ScanForCommittedOpIds(i, tablet_id, &index_to_term), Substitute("Couldn't scan log for TS $0", i)); for (const auto& index_term : index_to_term) { all_op_indexes.insert(index_term.first); @@ -179,7 +160,7 @@ Status LogVerifier::VerifyCommittedOpIdsMatch() { for (int i = 0; i < cluster_->num_tablet_servers(); i++) { if (i != 0) err += ", "; strings::SubstituteAndAppend(&err, "T $0=$1", - cluster_->tablet_server(i)->uuid(), + cluster_->UuidForTS(i), committed_terms[i]); } err += "]"; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/log_verifier.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/log_verifier.h b/src/kudu/integration-tests/log_verifier.h index 7f3d293..d831e6a 100644 --- a/src/kudu/integration-tests/log_verifier.h +++ b/src/kudu/integration-tests/log_verifier.h @@ -26,21 +26,24 @@ namespace kudu { +class Env; + namespace cluster { -class ExternalMiniCluster; -class ExternalTabletServer; +class MiniCluster; } // namespace cluster -class FsManager; - namespace consensus { class OpId; } // namespace consensus +namespace itest { +class MiniClusterFsInspector; +} + // Verifies correctness of the logs in an external mini-cluster. class LogVerifier { public: - explicit LogVerifier(cluster::ExternalMiniCluster* cluster); + explicit LogVerifier(cluster::MiniCluster* cluster); ~LogVerifier(); // Verify that, for every tablet in the cluster, the logs of each of that tablet's replicas @@ -58,21 +61,19 @@ class LogVerifier { // Scans the WAL on the given tablet server to find the COMMIT message with the highest // index. - Status ScanForHighestCommittedOpIdInLog(cluster::ExternalTabletServer* ets, + Status ScanForHighestCommittedOpIdInLog(int ts_idx, const std::string& tablet_id, consensus::OpId* commit_id); private: - // Open an FsManager for the given tablet server. - Status OpenFsManager(cluster::ExternalTabletServer* ets, - std::unique_ptr<FsManager>* fs); - - // Scan the WALs for tablet 'tablet_id' on the given 'fs'. Sets entries - // in '*index_to_term' for each COMMIT entry found in the WALs. - Status ScanForCommittedOpIds(FsManager* fs, const std::string& tablet_id, + // Scan the WALs for tablet 'tablet_id' on the server specified by 'ts_idx'. + // Sets entries in '*index_to_term' for each COMMIT entry found in the WALs. + Status ScanForCommittedOpIds(int ts_idx, const std::string& tablet_id, std::map<int64_t, int64_t>* index_to_term); - cluster::ExternalMiniCluster* const cluster_; + cluster::MiniCluster* const cluster_; + Env* const env_; + std::unique_ptr<itest::MiniClusterFsInspector> inspector_; DISALLOW_COPY_AND_ASSIGN(LogVerifier); }; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/mini_cluster_fs_inspector.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/mini_cluster_fs_inspector.cc new file mode 100644 index 0000000..8c186a5 --- /dev/null +++ b/src/kudu/integration-tests/mini_cluster_fs_inspector.cc @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" + +#include <algorithm> +#include <set> + +#include <glog/logging.h> + +#include "kudu/consensus/metadata.pb.h" +#include "kudu/fs/fs_manager.h" +#include "kudu/gutil/strings/join.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/gutil/strings/util.h" +#include "kudu/mini-cluster/mini_cluster.h" +#include "kudu/util/env.h" +#include "kudu/util/env_util.h" +#include "kudu/util/monotime.h" +#include "kudu/util/path_util.h" +#include "kudu/util/pb_util.h" +#include "kudu/util/status.h" + +namespace kudu { +namespace itest { + +using cluster::MiniCluster; +using std::set; +using std::string; +using std::vector; +using consensus::ConsensusMetadataPB; +using env_util::ListFilesInDir; +using strings::Substitute; +using tablet::TabletDataState; +using tablet::TabletSuperBlockPB; + +MiniClusterFsInspector::MiniClusterFsInspector(MiniCluster* cluster) + : cluster_(cluster), + env_(cluster->env()) {} + +int MiniClusterFsInspector::CountFilesInDir(const string& path, + StringPiece pattern) { + vector<string> entries; + Status s = ListFilesInDir(env_, path, &entries); + if (!s.ok()) return 0; + return std::count_if(entries.begin(), entries.end(), [&](const string& s) { + return pattern.empty() || MatchPattern(s, pattern); + }); +} + +string MiniClusterFsInspector::WalDirForTS(int ts_idx) const { + return JoinPathSegments(cluster_->WalRootForTS(ts_idx), FsManager::kWalDirName); +} + +int MiniClusterFsInspector::CountWALFilesOnTS(int ts_idx) { + string ts_wal_dir = WalDirForTS(ts_idx); + vector<string> tablets; + CHECK_OK(ListFilesInDir(env_, ts_wal_dir, &tablets)); + int total_segments = 0; + for (const string& tablet : tablets) { + string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet); + total_segments += CountFilesInDir(tablet_wal_dir); + } + return total_segments; +} + +vector<string> MiniClusterFsInspector::ListTablets() { + set<string> tablets; + for (int i = 0; i < cluster_->num_tablet_servers(); i++) { + auto ts_tablets = ListTabletsOnTS(i); + tablets.insert(ts_tablets.begin(), ts_tablets.end()); + } + return vector<string>(tablets.begin(), tablets.end()); +} + +vector<string> MiniClusterFsInspector::ListTabletsOnTS(int ts_idx) { + string meta_dir = JoinPathSegments(cluster_->WalRootForTS(ts_idx), + FsManager::kTabletMetadataDirName); + vector<string> tablets; + CHECK_OK(ListFilesInDir(env_, meta_dir, &tablets)); + return tablets; +} + +vector<string> MiniClusterFsInspector::ListTabletsWithDataOnTS(int ts_idx) { + vector<string> tablets; + CHECK_OK(ListFilesInDir(env_, WalDirForTS(ts_idx), &tablets)); + return tablets; +} + +int MiniClusterFsInspector::CountFilesInWALDirForTS( + int ts_idx, + const string& tablet_id, + StringPiece pattern) { + string tablet_wal_dir = JoinPathSegments(WalDirForTS(ts_idx), tablet_id); + if (!env_->FileExists(tablet_wal_dir)) { + return 0; + } + return CountFilesInDir(tablet_wal_dir, pattern); +} + +bool MiniClusterFsInspector::DoesConsensusMetaExistForTabletOnTS(int ts_idx, + const string& tablet_id) { + ConsensusMetadataPB cmeta_pb; + Status s = ReadConsensusMetadataOnTS(ts_idx, tablet_id, &cmeta_pb); + return s.ok(); +} + +int MiniClusterFsInspector::CountReplicasInMetadataDirs() { + // Rather than using FsManager's functionality for listing blocks, we just manually + // list the contents of the metadata directory. This is because we're using an + // external minicluster, and initializing a new FsManager to point at the running + // tablet servers isn't easy. + int count = 0; + for (int i = 0; i < cluster_->num_tablet_servers(); i++) { + count += CountFilesInDir(JoinPathSegments(cluster_->WalRootForTS(i), + FsManager::kTabletMetadataDirName)); + } + return count; +} + +Status MiniClusterFsInspector::CheckNoDataOnTS(int ts_idx) { + const string& wal_root = cluster_->WalRootForTS(ts_idx); + if (CountFilesInDir(JoinPathSegments(wal_root, FsManager::kTabletMetadataDirName)) > 0) { + return Status::IllegalState("tablet metadata blocks still exist", wal_root); + } + if (CountWALFilesOnTS(ts_idx) > 0) { + return Status::IllegalState("wals still exist", wal_root); + } + if (CountFilesInDir(JoinPathSegments(wal_root, FsManager::kConsensusMetadataDirName)) > 0) { + return Status::IllegalState("consensus metadata still exists", wal_root); + } + return Status::OK();; +} + +Status MiniClusterFsInspector::CheckNoData() { + for (int i = 0; i < cluster_->num_tablet_servers(); i++) { + RETURN_NOT_OK(CheckNoDataOnTS(i)); + } + return Status::OK();; +} + +string MiniClusterFsInspector::GetTabletSuperBlockPathOnTS(int ts_idx, + const string& tablet_id) const { + string meta_dir = JoinPathSegments(cluster_->WalRootForTS(ts_idx), + FsManager::kTabletMetadataDirName); + return JoinPathSegments(meta_dir, tablet_id); +} + +Status MiniClusterFsInspector::ReadTabletSuperBlockOnTS(int ts_idx, + const string& tablet_id, + TabletSuperBlockPB* sb) { + const auto& sb_path = GetTabletSuperBlockPathOnTS(ts_idx, tablet_id); + return pb_util::ReadPBContainerFromPath(env_, sb_path, sb); +} + +int64_t MiniClusterFsInspector::GetTabletSuperBlockMTimeOrDie(int ts_idx, + const string& tablet_id) { + int64_t timestamp; + CHECK_OK(env_->GetFileModifiedTime( + GetTabletSuperBlockPathOnTS(ts_idx, tablet_id), ×tamp)); + return timestamp; +} + +string MiniClusterFsInspector::GetConsensusMetadataPathOnTS(int ts_idx, + const string& tablet_id) const { + string wal_root = cluster_->WalRootForTS(ts_idx); + string cmeta_dir = JoinPathSegments(wal_root, FsManager::kConsensusMetadataDirName); + return JoinPathSegments(cmeta_dir, tablet_id); +} + +Status MiniClusterFsInspector::ReadConsensusMetadataOnTS(int ts_idx, + const string& tablet_id, + ConsensusMetadataPB* cmeta_pb) { + auto cmeta_path = GetConsensusMetadataPathOnTS(ts_idx, tablet_id); + if (!env_->FileExists(cmeta_path)) { + return Status::NotFound("Consensus metadata file not found", cmeta_path); + } + return pb_util::ReadPBContainerFromPath(env_, cmeta_path, cmeta_pb); +} + +Status MiniClusterFsInspector::WriteConsensusMetadataOnTS( + int ts_idx, + const string& tablet_id, + const ConsensusMetadataPB& cmeta_pb) { + auto cmeta_path = GetConsensusMetadataPathOnTS(ts_idx, tablet_id); + return pb_util::WritePBContainerToPath(env_, cmeta_path, cmeta_pb, + pb_util::OVERWRITE, pb_util::NO_SYNC); +} + + +Status MiniClusterFsInspector::CheckTabletDataStateOnTS( + int ts_idx, + const string& tablet_id, + const vector<TabletDataState>& allowed_states) { + TabletSuperBlockPB sb; + RETURN_NOT_OK(ReadTabletSuperBlockOnTS(ts_idx, tablet_id, &sb)); + if (std::find(allowed_states.begin(), allowed_states.end(), sb.tablet_data_state()) != + allowed_states.end()) { + return Status::OK(); + } + + vector<string> state_names; + for (auto state : allowed_states) { + state_names.push_back(TabletDataState_Name(state)); + } + string expected_str = JoinStrings(state_names, ","); + if (state_names.size() > 1) { + expected_str = "one of: " + expected_str; + } + + return Status::IllegalState(Substitute("State $0 unexpected, expected $1", + TabletDataState_Name(sb.tablet_data_state()), + expected_str)); +} + +Status MiniClusterFsInspector::WaitForNoData(const MonoDelta& timeout) { + MonoTime deadline = MonoTime::Now() + timeout; + Status s; + while (true) { + s = CheckNoData(); + if (s.ok()) return Status::OK(); + if (deadline < MonoTime::Now()) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(10)); + } + return Status::TimedOut("Timed out waiting for no data", s.ToString()); +} + +Status MiniClusterFsInspector::WaitForNoDataOnTS(int ts_idx, const MonoDelta& timeout) { + MonoTime deadline = MonoTime::Now() + timeout; + Status s; + while (true) { + s = CheckNoDataOnTS(ts_idx); + if (s.ok()) return Status::OK(); + if (deadline < MonoTime::Now()) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(10)); + } + return Status::TimedOut("Timed out waiting for no data", s.ToString()); +} + +Status MiniClusterFsInspector::WaitForMinFilesInTabletWalDirOnTS(int ts_idx, + const string& tablet_id, + int count, + const MonoDelta& timeout) { + int seen = 0; + MonoTime deadline = MonoTime::Now() + timeout; + while (true) { + seen = CountFilesInWALDirForTS(ts_idx, tablet_id); + if (seen >= count) return Status::OK(); + if (deadline < MonoTime::Now()) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(10)); + } + return Status::TimedOut(Substitute("Timed out waiting for number of WAL segments on tablet $0 " + "on TS $1 to be $2. Found $3", + tablet_id, ts_idx, count, seen)); +} + +Status MiniClusterFsInspector::WaitForReplicaCount(int expected, const MonoDelta& timeout) { + const MonoTime deadline = MonoTime::Now() + timeout; + int found; + while (true) { + found = CountReplicasInMetadataDirs(); + if (found == expected) { + return Status::OK(); + } + if (MonoTime::Now() > deadline) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(10)); + } + return Status::TimedOut( + Substitute("Timed out waiting for a total replica count of $0. " + "Found $1 replicas", expected, found)); +} + +Status MiniClusterFsInspector::WaitForTabletDataStateOnTS( + int ts_idx, + const string& tablet_id, + const vector<TabletDataState>& expected_states, + const MonoDelta& timeout) { + MonoTime start = MonoTime::Now(); + MonoTime deadline = start + timeout; + Status s; + while (true) { + s = CheckTabletDataStateOnTS(ts_idx, tablet_id, expected_states); + if (s.ok()) return Status::OK(); + if (MonoTime::Now() > deadline) break; + SleepFor(MonoDelta::FromMilliseconds(5)); + } + return Status::TimedOut(Substitute("Timed out after $0 waiting for correct tablet state: $1", + (MonoTime::Now() - start).ToString(), + s.ToString())); +} + +Status MiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs( + int ts_idx, const string& tablet_id, + const vector<string>& substrings_required, + const vector<string>& substrings_disallowed, + const MonoDelta& timeout) { + Status s; + MonoTime deadline = MonoTime::Now() + timeout; + + string tablet_wal_dir = JoinPathSegments(WalDirForTS(ts_idx), tablet_id); + + string error_msg; + vector<string> entries; + while (true) { + Status s = ListFilesInDir(env_, tablet_wal_dir, &entries); + std::sort(entries.begin(), entries.end()); + + error_msg = ""; + bool any_missing_required = false; + for (const string& required_filter : substrings_required) { + bool filter_matched = false; + for (const string& entry : entries) { + if (entry.find(required_filter) != string::npos) { + filter_matched = true; + break; + } + } + if (!filter_matched) { + any_missing_required = true; + error_msg += "missing from substrings_required: " + required_filter + "; "; + break; + } + } + + bool any_present_disallowed = false; + for (const string& entry : entries) { + if (any_present_disallowed) break; + for (const string& disallowed_filter : substrings_disallowed) { + if (entry.find(disallowed_filter) != string::npos) { + any_present_disallowed = true; + error_msg += Substitute("present from substrings_disallowed: $0 ($1)", + entry, disallowed_filter); + break; + } + } + } + + if (!any_missing_required && !any_present_disallowed) { + return Status::OK(); + } + if (MonoTime::Now() > deadline) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(10)); + } + + return Status::TimedOut(Substitute("Timed out waiting for file pattern on " + "tablet $0 on TS $1 in directory $2", + tablet_id, ts_idx, tablet_wal_dir), + error_msg + "entries: " + JoinStrings(entries, ", ")); +} + +} // namespace itest +} // namespace kudu + http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/mini_cluster_fs_inspector.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/mini_cluster_fs_inspector.h b/src/kudu/integration-tests/mini_cluster_fs_inspector.h new file mode 100644 index 0000000..ba41220 --- /dev/null +++ b/src/kudu/integration-tests/mini_cluster_fs_inspector.h @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include <cstdint> +#include <string> +#include <vector> + +#include "kudu/gutil/strings/stringpiece.h" +#include "kudu/tablet/metadata.pb.h" +#include "kudu/util/monotime.h" + +namespace kudu { + +class Env; +class Status; + +namespace cluster { +class MiniCluster; +} // namespace cluster + +namespace consensus { +class ConsensusMetadataPB; +} // namespace consensus + +namespace itest { + +// Utility class that digs around in a tablet server's FS layout and provides +// methods useful for integration testing. This class must outlive the Env and +// MiniCluster objects that are passed into it. +class MiniClusterFsInspector { + public: + explicit MiniClusterFsInspector(cluster::MiniCluster* cluster); + + ~MiniClusterFsInspector() {} + + // Returns the WALs FS subdirectory created for TS 'ts_idx'. + std::string WalDirForTS(int ts_idx) const; + + // If provided, files are filtered by the glob-style pattern 'pattern'. + int CountFilesInDir(const std::string& path, StringPiece pattern = StringPiece()); + + // List all of the tablets with tablet metadata in the cluster. + std::vector<std::string> ListTablets(); + + // List all of the tablets with tablet metadata on the given tablet server + // 'ts_idx'. This may include tablets that are tombstoned and not running. + std::vector<std::string> ListTabletsOnTS(int ts_idx); + + // List the tablet IDs on the given tablet which actually have data (as + // evidenced by their having a WAL). This excludes those that are tombstoned. + std::vector<std::string> ListTabletsWithDataOnTS(int ts_idx); + + // Returns the number of files in the WAL directory for 'tablet_id' on TS + // 'ts_idx'. If provided, files are filtered by the glob string 'pattern'. + int CountFilesInWALDirForTS(int ts_idx, + const std::string& tablet_id, + StringPiece pattern = StringPiece()); + + bool DoesConsensusMetaExistForTabletOnTS(int ts_idx, const std::string& tablet_id); + + int CountReplicasInMetadataDirs(); + Status CheckNoDataOnTS(int ts_idx); + Status CheckNoData(); + + Status ReadTabletSuperBlockOnTS(int ts_idx, const std::string& tablet_id, + tablet::TabletSuperBlockPB* sb); + + // Get the modification time (in micros) of the tablet superblock for the + // given tablet server ts_idx and tablet ID. + int64_t GetTabletSuperBlockMTimeOrDie(int ts_idx, const std::string& tablet_id); + + Status ReadConsensusMetadataOnTS(int ts_idx, const std::string& tablet_id, + consensus::ConsensusMetadataPB* cmeta_pb); + Status WriteConsensusMetadataOnTS(int ts_idx, + const std::string& tablet_id, + const consensus::ConsensusMetadataPB& cmeta_pb); + + Status CheckTabletDataStateOnTS(int ts_idx, + const std::string& tablet_id, + const std::vector<tablet::TabletDataState>& allowed_states); + + Status WaitForNoData(const MonoDelta& timeout = MonoDelta::FromSeconds(30)); + Status WaitForNoDataOnTS(int ts_idx, const MonoDelta& timeout = MonoDelta::FromSeconds(30)); + Status WaitForMinFilesInTabletWalDirOnTS(int ts_idx, + const std::string& tablet_id, + int count, + const MonoDelta& timeout = MonoDelta::FromSeconds(60)); + Status WaitForReplicaCount(int expected, const MonoDelta& timeout = MonoDelta::FromSeconds(30)); + Status WaitForTabletDataStateOnTS(int ts_idx, + const std::string& tablet_id, + const std::vector<tablet::TabletDataState>& expected_states, + const MonoDelta& timeout = MonoDelta::FromSeconds(30)); + + // Loop and check for certain filenames in the WAL directory of the specified + // tablet. This function returns OK if we reach a state where: + // * For each string in 'substrings_required', we find *at least one file* + // whose name contains that string, and: + // * For each string in 'substrings_disallowed', we find *no files* whose name + // contains that string, even if the file also matches a string in the + // 'substrings_required'. + Status WaitForFilePatternInTabletWalDirOnTs( + int ts_idx, + const std::string& tablet_id, + const std::vector<std::string>& substrings_required, + const std::vector<std::string>& substrings_disallowed, + const MonoDelta& timeout = MonoDelta::FromSeconds(30)); + + private: + const cluster::MiniCluster* const cluster_; + Env* const env_; + + // Return the number of files in WAL directories on the given tablet server. + // This includes log ts_idx files (not just segments). + int CountWALFilesOnTS(int ts_idx); + + std::string GetConsensusMetadataPathOnTS(int ts_idx, + const std::string& tablet_id) const; + + std::string GetTabletSuperBlockPathOnTS(int ts_idx, + const std::string& tablet_id) const; + +}; + +} // namespace itest +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/raft_config_change-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/raft_config_change-itest.cc b/src/kudu/integration-tests/raft_config_change-itest.cc index d7b1301..c17f5c6 100644 --- a/src/kudu/integration-tests/raft_config_change-itest.cc +++ b/src/kudu/integration-tests/raft_config_change-itest.cc @@ -37,7 +37,7 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/raft_consensus-itest-base.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/raft_consensus-itest-base.cc b/src/kudu/integration-tests/raft_consensus-itest-base.cc index 5b5686d..f10b652 100644 --- a/src/kudu/integration-tests/raft_consensus-itest-base.cc +++ b/src/kudu/integration-tests/raft_consensus-itest-base.cc @@ -41,7 +41,7 @@ #include "kudu/gutil/gscoped_ptr.h" #include "kudu/gutil/stringprintf.h" #include "kudu/integration-tests/cluster_itest_util.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/integration-tests/ts_itest-base.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/raft_consensus-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc index 7bd4f16..41193a8 100644 --- a/src/kudu/integration-tests/raft_consensus-itest.cc +++ b/src/kudu/integration-tests/raft_consensus-itest.cc @@ -54,8 +54,8 @@ #include "kudu/gutil/strings/util.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" #include "kudu/integration-tests/log_verifier.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/raft_consensus-itest-base.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" @@ -1064,10 +1064,11 @@ TEST_F(RaftConsensusITest, TestLMPMismatchOnRestartedReplica) { // The COMMIT messages end up in the WAL asynchronously, so loop reading the // tablet server's WAL until it shows up. + int replica_idx = cluster_->tablet_server_index_by_uuid(replica_ets->uuid()); ASSERT_EVENTUALLY([&]() { LogVerifier lv(cluster_.get()); OpId commit; - ASSERT_OK(lv.ScanForHighestCommittedOpIdInLog(replica_ets, tablet_id_, &commit)); + ASSERT_OK(lv.ScanForHighestCommittedOpIdInLog(replica_idx, tablet_id_, &commit)); ASSERT_EQ("2.2", OpIdToString(commit)); }); http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/raft_consensus_election-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/raft_consensus_election-itest.cc b/src/kudu/integration-tests/raft_consensus_election-itest.cc index cb8803b..00078bf 100644 --- a/src/kudu/integration-tests/raft_consensus_election-itest.cc +++ b/src/kudu/integration-tests/raft_consensus_election-itest.cc @@ -32,7 +32,7 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/raft_consensus-itest-base.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc index 6726109..8908029 100644 --- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc +++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc @@ -41,7 +41,7 @@ #include "kudu/gutil/strings/util.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/raft_consensus-itest-base.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/tablet_copy-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/tablet_copy-itest.cc b/src/kudu/integration-tests/tablet_copy-itest.cc index 3be6d0f..4b2a9cf 100644 --- a/src/kudu/integration-tests/tablet_copy-itest.cc +++ b/src/kudu/integration-tests/tablet_copy-itest.cc @@ -55,7 +55,7 @@ #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/tablet_copy_client_session-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/tablet_copy_client_session-itest.cc b/src/kudu/integration-tests/tablet_copy_client_session-itest.cc index 5279d0b..78a0e77 100644 --- a/src/kudu/integration-tests/tablet_copy_client_session-itest.cc +++ b/src/kudu/integration-tests/tablet_copy_client_session-itest.cc @@ -35,7 +35,7 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tablet/metadata.pb.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/tablet_replacement-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/tablet_replacement-itest.cc b/src/kudu/integration-tests/tablet_replacement-itest.cc index 7ca9c3b..5aeaed5 100644 --- a/src/kudu/integration-tests/tablet_replacement-itest.cc +++ b/src/kudu/integration-tests/tablet_replacement-itest.cc @@ -39,7 +39,7 @@ #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/rpc/rpc_controller.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/tombstoned_voting-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/tombstoned_voting-itest.cc b/src/kudu/integration-tests/tombstoned_voting-itest.cc index 951f28f..9c28cd9 100644 --- a/src/kudu/integration-tests/tombstoned_voting-itest.cc +++ b/src/kudu/integration-tests/tombstoned_voting-itest.cc @@ -29,7 +29,7 @@ #include "kudu/gutil/map-util.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" #include "kudu/mini-cluster/external_mini_cluster.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/tombstoned_voting-stress-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/tombstoned_voting-stress-test.cc b/src/kudu/integration-tests/tombstoned_voting-stress-test.cc index ba71f8e..e300630 100644 --- a/src/kudu/integration-tests/tombstoned_voting-stress-test.cc +++ b/src/kudu/integration-tests/tombstoned_voting-stress-test.cc @@ -36,7 +36,7 @@ #include "kudu/gutil/macros.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tablet/metadata.pb.h" http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/ts_itest-base.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_itest-base.cc b/src/kudu/integration-tests/ts_itest-base.cc index 26f4010..6ec2000 100644 --- a/src/kudu/integration-tests/ts_itest-base.cc +++ b/src/kudu/integration-tests/ts_itest-base.cc @@ -45,7 +45,7 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" #include "kudu/mini-cluster/external_mini_cluster.h" @@ -144,7 +144,7 @@ void TabletServerIntegrationTestBase::CreateCluster( cluster_.reset(new cluster::ExternalMiniCluster(std::move(opts))); ASSERT_OK(cluster_->Start()); - inspect_.reset(new itest::ExternalMiniClusterFsInspector(cluster_.get())); + inspect_.reset(new itest::MiniClusterFsInspector(cluster_.get())); CreateTSProxies(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/ts_itest-base.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_itest-base.h b/src/kudu/integration-tests/ts_itest-base.h index 19ee9c5..ce41df4 100644 --- a/src/kudu/integration-tests/ts_itest-base.h +++ b/src/kudu/integration-tests/ts_itest-base.h @@ -25,7 +25,7 @@ #include "kudu/client/shared_ptr.h" #include "kudu/gutil/gscoped_ptr.h" #include "kudu/integration-tests/cluster_itest_util.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tserver/tablet_server-test-base.h" #include "kudu/util/random.h" @@ -151,7 +151,7 @@ class TabletServerIntegrationTestBase : public TabletServerTestBase { protected: gscoped_ptr<cluster::ExternalMiniCluster> cluster_; - gscoped_ptr<itest::ExternalMiniClusterFsInspector> inspect_; + gscoped_ptr<itest::MiniClusterFsInspector> inspect_; // Maps server uuid to TServerDetails itest::TabletServerMap tablet_servers_; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/ts_recovery-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_recovery-itest.cc b/src/kudu/integration-tests/ts_recovery-itest.cc index c396bf8..4b78e15 100644 --- a/src/kudu/integration-tests/ts_recovery-itest.cc +++ b/src/kudu/integration-tests/ts_recovery-itest.cc @@ -58,7 +58,7 @@ #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tablet/metadata.pb.h" @@ -101,7 +101,7 @@ using consensus::ConsensusMetadataManager; using consensus::OpId; using consensus::RECEIVED_OPID; using fs::BlockManager; -using itest::ExternalMiniClusterFsInspector; +using itest::MiniClusterFsInspector; using log::AppendNoOpsToLogSync; using log::Log; using log::LogOptions; @@ -164,8 +164,8 @@ TEST_P(TsRecoveryITest, TestNoBlockIDReuseIfMissingBlocks) { }; unique_ptr<TestWorkload> write_workload(StartSingleTabletWorkload("foo")); - unique_ptr<ExternalMiniClusterFsInspector> inspect( - new ExternalMiniClusterFsInspector(cluster_.get())); + unique_ptr<MiniClusterFsInspector> inspect( + new MiniClusterFsInspector(cluster_.get())); vector<string> tablets = inspect->ListTabletsOnTS(0); ASSERT_EQ(1, tablets.size()); const string tablet_id = tablets[0]; http://git-wip-us.apache.org/repos/asf/kudu/blob/2165ce57/src/kudu/integration-tests/ts_tablet_manager-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/ts_tablet_manager-itest.cc b/src/kudu/integration-tests/ts_tablet_manager-itest.cc index e56d016..8efebe9 100644 --- a/src/kudu/integration-tests/ts_tablet_manager-itest.cc +++ b/src/kudu/integration-tests/ts_tablet_manager-itest.cc @@ -39,6 +39,7 @@ #include "kudu/gutil/stl_util.h" #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" +#include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" @@ -86,6 +87,7 @@ using kudu::rpc::Messenger; using kudu::rpc::MessengerBuilder; using kudu::tablet::TabletReplica; using kudu::tserver::MiniTabletServer; +using kudu::ClusterVerifier; using std::map; using std::shared_ptr; using std::string; @@ -213,28 +215,11 @@ TEST_P(FailedTabletsAreReplacedITest, OneReplica) { ASSERT_EQ(1, tablet_ids.size()); tablet_id = tablet_ids[0]; }); + work.StopAndJoin(); // Wait until all the replicas are running before failing one arbitrarily. - const auto wait_until_running = [&]() { - AssertEventually([&]{ - auto num_replicas_running = 0; - for (auto idx = 0; idx < cluster_->num_tablet_servers(); ++idx) { - MiniTabletServer* ts = cluster_->mini_tablet_server(idx); - scoped_refptr<TabletReplica> replica; - Status s = ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica); - if (s.IsNotFound()) { - continue; - } - ASSERT_OK(s); - if (tablet::RUNNING == replica->state()) { - ++num_replicas_running; - } - } - ASSERT_EQ(kNumReplicas, num_replicas_running); - }, MonoDelta::FromSeconds(60)); - NO_PENDING_FATALS(); - }; - NO_FATALS(wait_until_running()); + ClusterVerifier v(cluster_.get()); + NO_FATALS(v.CheckCluster()); { // Inject an error into one of replicas. Shutting it down will leave it in @@ -251,8 +236,7 @@ TEST_P(FailedTabletsAreReplacedITest, OneReplica) { } // Ensure the tablet eventually is replicated. - NO_FATALS(wait_until_running()); - work.StopAndJoin(); + NO_FATALS(v.CheckCluster()); } INSTANTIATE_TEST_CASE_P(, FailedTabletsAreReplacedITest,
