This is an automated email from the ASF dual-hosted git repository.
zhangyifan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 81f7c5b43 [Tools] Rebuild master according to part of tables not all
tables
81f7c5b43 is described below
commit 81f7c5b43fe4546ee1b7526b51a13126da706980
Author: xinghuayu007 <[email protected]>
AuthorDate: Thu May 26 09:52:28 2022 +0800
[Tools] Rebuild master according to part of tables not all tables
Current function unsafe_rebuild will get all tables's meta data
from tservers, and rebuild a new sys table. The cost is very
high when only part of tables got damaged.
This patch will rebuild master with configured tables. That
means if some tables meta data get damaged, it only needs to
repair these damaged tables meta data.
Change-Id: I45d812c8a16228ac937603872161969eb05136eb
Reviewed-on: http://gerrit.cloudera.org:8080/18572
Reviewed-by: Yifan Zhang <[email protected]>
Tested-by: Yifan Zhang <[email protected]>
---
src/kudu/master/sys_catalog-test.cc | 45 +---------
src/kudu/master/sys_catalog.cc | 30 +++++++
src/kudu/master/sys_catalog.h | 27 ++++++
src/kudu/tools/kudu-admin-test.cc | 159 ++++++++++++++++++++++++++++++++---
src/kudu/tools/master_rebuilder.cc | 148 +++++++++++++++++++++++++++-----
src/kudu/tools/master_rebuilder.h | 6 +-
src/kudu/tools/tool_action_master.cc | 7 +-
7 files changed, 342 insertions(+), 80 deletions(-)
diff --git a/src/kudu/master/sys_catalog-test.cc
b/src/kudu/master/sys_catalog-test.cc
index 67957c7cb..e28a10625 100644
--- a/src/kudu/master/sys_catalog-test.cc
+++ b/src/kudu/master/sys_catalog-test.cc
@@ -95,26 +95,6 @@ class SysCatalogTest : public KuduTest {
unique_ptr<MasterServiceProxy> proxy_;
};
-class TestTableLoader : public TableVisitor {
- public:
- void Reset() {
- tables.clear();
- }
-
- Status VisitTable(const string& table_id,
- const SysTablesEntryPB& metadata) override {
- // Setup the table info
- scoped_refptr<TableInfo> table = new TableInfo(table_id);
- TableMetadataLock l(table.get(), LockMode::WRITE);
- l.mutable_data()->pb.CopyFrom(metadata);
- l.Commit();
- tables.emplace_back(std::move(table));
- return Status::OK();
- }
-
- vector<scoped_refptr<TableInfo>> tables;
-};
-
static bool PbEquals(const google::protobuf::Message& a, const
google::protobuf::Message& b) {
return pb_util::SecureDebugString(a) == pb_util::SecureDebugString(b);
}
@@ -130,7 +110,7 @@ static bool MetadatasEqual(const scoped_refptr<C>& ti_a,
// Test the sys-catalog tables basic operations (add, update, delete,
// visit)
TEST_F(SysCatalogTest, TestSysCatalogTablesOperations) {
- TestTableLoader loader;
+ TableInfoLoader loader;
auto* sys_catalog = master_->catalog_manager()->sys_catalog();
ASSERT_OK(sys_catalog->VisitTables(&loader));
@@ -226,27 +206,6 @@ TEST_F(SysCatalogTest, TestTableInfoCommit) {
}
}
-class TestTabletLoader : public TabletVisitor {
- public:
- void Reset() {
- tablets.clear();
- }
-
- Status VisitTablet(const string& /*table_id*/,
- const string& tablet_id,
- const SysTabletsEntryPB& metadata) override {
- // Setup the tablet info
- scoped_refptr<TabletInfo> tablet = new TabletInfo(nullptr, tablet_id);
- TabletMetadataLock l(tablet.get(), LockMode::WRITE);
- l.mutable_data()->pb.CopyFrom(metadata);
- l.Commit();
- tablets.emplace_back(std::move(tablet));
- return Status::OK();
- }
-
- vector<scoped_refptr<TabletInfo>> tablets;
-};
-
// Create a new TabletInfo. The object is in uncommitted
// state.
static scoped_refptr<TabletInfo> CreateTablet(
@@ -274,7 +233,7 @@ TEST_F(SysCatalogTest, TestSysCatalogTabletsOperations) {
SysCatalogTable* sys_catalog = master_->catalog_manager()->sys_catalog();
- TestTabletLoader loader;
+ TabletInfoLoader loader;
ASSERT_OK(master_->catalog_manager()->sys_catalog()->VisitTablets(&loader));
ASSERT_EQ(0, loader.tablets.size());
diff --git a/src/kudu/master/sys_catalog.cc b/src/kudu/master/sys_catalog.cc
index 58a44922b..1c346cacf 100644
--- a/src/kudu/master/sys_catalog.cc
+++ b/src/kudu/master/sys_catalog.cc
@@ -1157,5 +1157,35 @@ void SysCatalogTable::InitLocalRaftPeerPB() {
*local_peer_pb_.mutable_last_known_addr() = HostPortToPB(hps[0]);
}
+void TableInfoLoader::Reset() {
+ tables.clear();
+}
+
+Status TableInfoLoader::VisitTable(const string& table_id,
+ const SysTablesEntryPB& metadata) {
+ // Setup the table info
+ scoped_refptr<TableInfo> table = new TableInfo(table_id);
+ TableMetadataLock l(table.get(), LockMode::WRITE);
+ l.mutable_data()->pb.CopyFrom(metadata);
+ l.Commit();
+ tables.emplace_back(std::move(table));
+ return Status::OK();
+}
+
+void TabletInfoLoader::Reset() {
+ tablets.clear();
+}
+
+Status TabletInfoLoader::VisitTablet(const string& /*table_id*/,
+ const string& tablet_id,
+ const SysTabletsEntryPB& metadata) {
+ // Setup the tablet info
+ scoped_refptr<TabletInfo> tablet = new TabletInfo(nullptr, tablet_id);
+ TabletMetadataLock l(tablet.get(), LockMode::WRITE);
+ l.mutable_data()->pb.CopyFrom(metadata);
+ l.Commit();
+ tablets.emplace_back(std::move(tablet));
+ return Status::OK();
+}
} // namespace master
} // namespace kudu
diff --git a/src/kudu/master/sys_catalog.h b/src/kudu/master/sys_catalog.h
index a82829ff1..b4d6dd11e 100644
--- a/src/kudu/master/sys_catalog.h
+++ b/src/kudu/master/sys_catalog.h
@@ -101,6 +101,27 @@ class TServerStateVisitor {
const SysTServerStateEntryPB& metadata) = 0;
};
+class TableInfoLoader : public TableVisitor {
+ public:
+ void Reset();
+
+ Status VisitTable(const std::string& table_id,
+ const SysTablesEntryPB& metadata) override;
+
+ std::vector<scoped_refptr<TableInfo>> tables;
+};
+
+class TabletInfoLoader : public TabletVisitor {
+ public:
+ void Reset();
+
+ Status VisitTablet(const std::string& /*table_id*/,
+ const std::string& tablet_id,
+ const SysTabletsEntryPB& metadata) override;
+
+ std::vector<scoped_refptr<TabletInfo>> tablets;
+};
+
// SysCatalogTable is a Kudu table that keeps track of the following
// system information:
// * cluster id
@@ -157,6 +178,12 @@ class SysCatalogTable {
CLUSTER_ID = 7 // Unique Cluster ID.
};
+ enum SysCatalogOperation {
+ ADD,
+ UPDATE,
+ DELETE,
+ };
+
// 'leader_cb_' is invoked whenever this node is elected as a leader
// of the consensus configuration for this tablet, including for local
standalone
// master consensus configurations. It used to initialize leader state,
submit any
diff --git a/src/kudu/tools/kudu-admin-test.cc
b/src/kudu/tools/kudu-admin-test.cc
index 120b0da84..3b9b12d62 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -16,7 +16,6 @@
// under the License.
#include <algorithm>
-#include <atomic>
#include <cstdint>
#include <cstdio>
#include <deque>
@@ -50,8 +49,11 @@
#include "kudu/consensus/metadata.pb.h"
#include "kudu/consensus/opid.pb.h"
#include "kudu/consensus/quorum_util.h"
+#include "kudu/consensus/raft_consensus.h"
+#include "kudu/fs/fs_manager.h"
#include "kudu/gutil/basictypes.h"
#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/stl_util.h"
#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/split.h"
@@ -61,12 +63,17 @@
#include "kudu/integration-tests/cluster_verifier.h"
#include "kudu/integration-tests/test_workload.h"
#include "kudu/integration-tests/ts_itest-base.h"
+#include "kudu/master/catalog_manager.h"
+#include "kudu/master/master.h"
#include "kudu/master/master.pb.h"
+#include "kudu/master/master_options.h"
#include "kudu/master/sys_catalog.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
#include "kudu/tablet/metadata.pb.h"
+#include "kudu/tablet/tablet_replica.h"
#include "kudu/tools/tool_test_util.h"
#include "kudu/tserver/tablet_server-test-base.h"
+#include "kudu/util/cow_object.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
@@ -76,12 +83,6 @@
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
-namespace kudu {
-namespace tserver {
-class ListTabletsResponsePB;
-} // namespace tserver
-} // namespace kudu
-
DECLARE_int32(num_replicas);
DECLARE_int32(num_tablet_servers);
DECLARE_string(sasl_protocol_name);
@@ -118,13 +119,19 @@ using kudu::itest::WAIT_FOR_LEADER;
using kudu::itest::WaitForReplicasReportedToMaster;
using kudu::itest::WaitForServersToAgree;
using kudu::itest::WaitUntilCommittedConfigNumVotersIs;
-using kudu::itest::WaitUntilCommittedOpIdIndexIs;
using kudu::itest::WaitUntilTabletInState;
using kudu::itest::WaitUntilTabletRunning;
+using kudu::master::Master;
+using kudu::master::MasterOptions;
+using kudu::master::SysCatalogTable;
+using kudu::master::TableInfo;
+using kudu::master::TableInfoLoader;
+using kudu::master::TableMetadataLock;
+using kudu::master::TabletInfo;
+using kudu::master::TabletInfoLoader;
+using kudu::master::TabletMetadataGroupLock;
using kudu::master::VOTER_REPLICA;
using kudu::pb_util::SecureDebugString;
-using kudu::tserver::ListTabletsResponsePB;
-using std::atomic;
using std::back_inserter;
using std::copy;
using std::deque;
@@ -142,7 +149,13 @@ namespace kudu {
namespace tools {
- // Helper to format info when a tool action fails.
+namespace {
+Status NoOpCb() {
+ return Status::OK();
+}
+} // anonymous namespace
+
+// Helper to format info when a tool action fails.
static string ToolRunInfo(const Status& s, const string& out, const string&
err) {
ostringstream str;
str << s.ToString() << endl;
@@ -3038,7 +3051,10 @@ constexpr const char* kPrincipal = "oryx";
vector<string> RebuildMasterCmd(const ExternalMiniCluster& cluster,
int tserver_num,
- bool is_secure, bool log_to_stderr = false) {
+ bool is_secure,
+ bool log_to_stderr = false,
+ const string& tables = "",
+ const int& default_replica_num = 1) {
CHECK_GT(tserver_num, 0);
CHECK_LE(tserver_num, cluster.num_tablet_servers());
vector<string> command = {
@@ -3049,6 +3065,9 @@ vector<string> RebuildMasterCmd(const
ExternalMiniCluster& cluster,
"-fs_wal_dir",
cluster.master()->wal_dir(),
};
+ if (!tables.empty()) {
+ command.emplace_back(Substitute("-tables=$0", tables));
+ }
if (log_to_stderr) {
command.emplace_back("--logtostderr");
}
@@ -3059,6 +3078,7 @@ vector<string> RebuildMasterCmd(const
ExternalMiniCluster& cluster,
auto* ts = cluster.tablet_server(i);
command.emplace_back(ts->bound_rpc_hostport().ToString());
}
+ command.emplace_back(Substitute("--default_num_replicas=$0",
default_replica_num));
return command;
}
@@ -3106,6 +3126,118 @@ TEST_F(AdminCliTest, TestRebuildMasterWhenNonEmpty) {
ASSERT_STR_CONTAINS(stdout, "Rebuilt from 1 replicas, of which 0 had
errors");
}
+void delete_table_in_syscatalog(const string& wal_dir,
+ const std::vector<std::string>& data_dirs,
+ const string& del_table_name) {
+ MasterOptions opts;
+ opts.fs_opts.wal_root = wal_dir;
+ opts.fs_opts.data_roots = data_dirs;
+ Master master(opts);
+ ASSERT_OK(master.Init());
+ SysCatalogTable sys_catalog(&master, &NoOpCb);
+ ASSERT_OK(sys_catalog.Load(master.fs_manager()));
+ // Get table from sys_catalog.
+ const auto kLeaderTimeout = MonoDelta::FromSeconds(10);
+
ASSERT_OK(sys_catalog.tablet_replica()->consensus()->WaitUntilLeader(kLeaderTimeout));
+ TableInfoLoader table_info_loader;
+ sys_catalog.VisitTables(&table_info_loader);
+ scoped_refptr<TableInfo> table_info;
+ for (const auto& table : table_info_loader.tables) {
+ table->metadata().ReadLock();
+ string table_name = table->metadata().state().name();
+ table->metadata().ReadUnlock();
+ if (table_name == del_table_name) {
+ table_info = table;
+ break;
+ }
+ }
+ ASSERT_NE(nullptr, table_info);
+
+ // Get tablet from sys_catalog.
+ TabletInfoLoader tablet_info_loader;
+ sys_catalog.VisitTablets(&tablet_info_loader);
+ vector<scoped_refptr<TabletInfo>> tablets;
+ for (const auto& tablet : tablet_info_loader.tablets) {
+ tablet->metadata().ReadLock();
+ if (tablet->metadata().state().pb.table_id() == table_info->id())
+ tablets.push_back(tablet);
+ tablet->metadata().ReadUnlock();
+ }
+ ASSERT_GT(tablets.size(), 0);
+
+ // Delete one table and it's tablets.
+ TableMetadataLock l_table(table_info.get(), LockMode::WRITE);
+ TabletMetadataGroupLock l_tablets(LockMode::RELEASED);
+ l_tablets.AddMutableInfos(tablets);
+ l_tablets.Lock(LockMode::WRITE);
+ SysCatalogTable::Actions actions;
+ actions.table_to_delete = table_info;
+ actions.tablets_to_delete = tablets;
+ ASSERT_OK(sys_catalog.Write(actions));
+
+ NO_FATALS(sys_catalog.Shutdown());
+ NO_FATALS(master.Shutdown());
+}
+
+// Rebuild tables according to part of tables not all tables.
+TEST_F(AdminCliTest, TestRebuildTables) {
+ FLAGS_num_tablet_servers = 3;
+ NO_FATALS(BuildAndStart({}, {}, {}, /*create_table*/false));
+ // Create 3 tables.
+ constexpr const char* kTable1 = "TestTable";
+ NO_FATALS(MakeTestTable(kTable1, /*num_rows*/10, /*num_replicas*/1,
cluster_.get()));
+ constexpr const char* kTable2 = "TestTable1";
+ NO_FATALS(MakeTestTable(kTable2, /*num_rows*/10, /*num_replicas*/1,
cluster_.get()));
+ constexpr const char* kTable3 = "TestTable2";
+ NO_FATALS(MakeTestTable(kTable3, /*num_rows*/10, /*num_replicas*/1,
cluster_.get()));
+
+ const string& part_tables = Substitute("$0,$1", kTable1, kTable2);
+ NO_FATALS(cluster_->master()->Shutdown());
+
+ string stdout1;
+ string stderr1;
+ // Rebuild 2 tables in update mode.
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, FLAGS_num_tablet_servers,
+ /*is_secure*/false,
/*log_to_stderr*/true,
+ part_tables,
/*default_replica_num*/1),
+ &stdout1, &stderr1));
+ ASSERT_STR_CONTAINS(stdout1,
+ "Rebuilt from 3 tablet servers, of which 0 had errors");
+ ASSERT_STR_CONTAINS(stdout1, "Rebuilt from 2 replicas, of which 0 had
errors");
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ cluster_->tablet_server(i)->Shutdown();
+ }
+ // Restart the cluster to check cluster healthy.
+ cluster_->Restart();
+ WaitForTSAndReplicas();
+ ClusterVerifier cv1(cluster_.get());
+ NO_FATALS(cv1.CheckCluster());
+
+ NO_FATALS(cluster_->master()->Shutdown());
+ // Delete kTable1 in syscatalog.
+ delete_table_in_syscatalog(cluster_->master()->wal_dir(),
+ cluster_->master()->data_dirs(),
+ kTable1);
+ string stdout2;
+ string stderr2;
+ // Rebuild kTable1 in add mode.
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, FLAGS_num_tablet_servers,
+ /*is_secure*/false,
/*log_to_stderr*/true, kTable1),
+ &stdout2, &stderr2));
+ ASSERT_STR_NOT_CONTAINS(stderr2, "must be empty");
+ ASSERT_STR_CONTAINS(stdout2,
+ "Rebuilt from 3 tablet servers, of which 0 had errors");
+ ASSERT_STR_CONTAINS(stdout2, "Rebuilt from 1 replicas, of which 0 had
errors");
+ for (int i = 0; i < cluster_->num_tablet_servers(); i++) {
+ cluster_->tablet_server(i)->Shutdown();
+ }
+ // Restart the cluster to check cluster healthy.
+ cluster_->Restart();
+ WaitForTSAndReplicas();
+ ClusterVerifier cv2(cluster_.get());
+ NO_FATALS(cv2.CheckCluster());
+}
+
// Test that the master rebuilder ignores tombstones.
TEST_F(AdminCliTest, TestRebuildMasterWithTombstones) {
FLAGS_num_tablet_servers = 3;
@@ -3203,7 +3335,8 @@ TEST_F(AdminCliTest, TestAddColumnsAndRebuildMaster) {
// The tool will firstly use schema on tserver-0 which holds an outdated
schema, then
// use the newer schema on tserver-1 to rebuild master.
string stdout;
- ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, 2, /*is_secure*/false,
/*log_to_stderr*/true),
+ ASSERT_OK(RunKuduTool(RebuildMasterCmd(*cluster_, 2,
+ /*is_secure*/false, /*log_to_stderr*/true, "", 3),
&stdout));
ASSERT_STR_CONTAINS(stdout, "Rebuilt from 2 tablet servers, of which 0 had
errors");
ASSERT_STR_CONTAINS(stdout, "Rebuilt from 2 replicas, of which 0 had
errors");
diff --git a/src/kudu/tools/master_rebuilder.cc
b/src/kudu/tools/master_rebuilder.cc
index 2a21d99b9..a6cb1ca53 100644
--- a/src/kudu/tools/master_rebuilder.cc
+++ b/src/kudu/tools/master_rebuilder.cc
@@ -22,6 +22,7 @@
#include <map>
#include <memory>
#include <ostream>
+#include <set>
#include <string>
#include <vector>
@@ -37,6 +38,7 @@
#include "kudu/consensus/raft_consensus.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/split.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/master/catalog_manager.h"
#include "kudu/master/master.h"
@@ -62,6 +64,7 @@ DEFINE_uint32(default_schema_version, 0, "The table schema
version assigned to t
"'kudu pbc dump tablet-meta/<tablet-id>' on each server and
taking the max value "
"found across all tablets. In tablet server versions >= 1.16,
Kudu will determine "
"the proper value for each tablet automatically.");
+DECLARE_string(tables);
using kudu::master::Master;
using kudu::master::MasterOptions;
@@ -69,11 +72,15 @@ using kudu::master::SysCatalogTable;
using kudu::master::SysTablesEntryPB;
using kudu::master::SysTabletsEntryPB;
using kudu::master::TableInfo;
+using kudu::master::TableInfoLoader;
using kudu::master::TableMetadataLock;
using kudu::master::TabletInfo;
+using kudu::master::TabletInfoLoader;
using kudu::master::TabletMetadataGroupLock;
using kudu::master::TabletMetadataLock;
using kudu::tserver::ListTabletsResponsePB;
+using std::map;
+using std::set;
using std::string;
using std::vector;
using strings::Substitute;
@@ -87,6 +94,58 @@ Status NoOpCb() {
}
} // anonymous namespace
+static Status DoWrite(SysCatalogTable* sys_catalog,
+ const map<string, scoped_refptr<master::TableInfo>>&
table_by_name,
+ SysCatalogTable::SysCatalogOperation operation,
+ map<string, vector<scoped_refptr<TabletInfo>>>*
tablet_by_name = nullptr) {
+ if (table_by_name.empty()) return Status::OK();
+ const auto kLeaderTimeout = MonoDelta::FromSeconds(10);
+
RETURN_NOT_OK(sys_catalog->tablet_replica()->consensus()->WaitUntilLeader(kLeaderTimeout));
+ for (const auto& table_entry : table_by_name) {
+ const auto& table = table_entry.second;
+ vector<scoped_refptr<TabletInfo>> tablets;
+ if (tablet_by_name == nullptr ||
+ tablet_by_name->empty()) {
+ table->GetAllTablets(&tablets);
+ } else {
+ if (ContainsKey(*tablet_by_name, table_entry.first))
+ tablets = (*tablet_by_name)[table_entry.first];
+ }
+ TableMetadataLock l_table(table.get(), LockMode::WRITE);
+ TabletMetadataGroupLock l_tablets(LockMode::RELEASED);
+ l_tablets.AddMutableInfos(tablets);
+ l_tablets.Lock(LockMode::WRITE);
+ SysCatalogTable::Actions actions;
+ switch (operation) {
+ case SysCatalogTable::SysCatalogOperation::ADD:
+ {
+ actions.table_to_add = table;
+ actions.tablets_to_add = tablets;
+ break;
+ }
+ case SysCatalogTable::SysCatalogOperation::UPDATE:
+ {
+ actions.table_to_update = table;
+ actions.tablets_to_update = tablets;
+ break;
+ }
+ case SysCatalogTable::SysCatalogOperation::DELETE:
+ {
+ actions.table_to_delete = table;
+ actions.tablets_to_delete = tablets;
+ break;
+ }
+ default:
+ return Status::InvalidArgument(Substitute("Operation:$ is not
supported.",
+ operation));
+ }
+ RETURN_NOT_OK_PREPEND(sys_catalog->Write(actions),
+ Substitute("unable to write metadata for table $0 to
sys_catalog",
+ table_entry.first));
+ }
+ return Status::OK();
+}
+
MasterRebuilder::MasterRebuilder(vector<string> tserver_addrs)
: state_(State::NOT_DONE),
tserver_addrs_(std::move(tserver_addrs)) {
@@ -101,6 +160,8 @@ Status MasterRebuilder::RebuildMaster() {
CHECK_EQ(State::NOT_DONE, state_);
int bad_tservers = 0;
+ const set<string>& filter_tables = strings::Split(FLAGS_tables, ",",
+ strings::SkipWhitespace());
for (const auto& tserver_addr : tserver_addrs_) {
std::unique_ptr<tserver::TabletServerServiceProxy> proxy;
vector<ListTabletsResponsePB::StatusAndSchemaPB> replicas;
@@ -121,6 +182,7 @@ Status MasterRebuilder::RebuildMaster() {
const auto& state_str = TabletStatePB_Name(state_pb);
const auto& tablet_id = tablet_status_pb.tablet_id();
const auto& table_name = tablet_status_pb.table_name();
+ if (!filter_tables.empty() && !ContainsKey(filter_tables, table_name))
continue;
switch (state_pb) {
case tablet::STOPPING:
case tablet::STOPPED:
@@ -157,10 +219,12 @@ Status MasterRebuilder::RebuildMaster() {
if (bad_tservers == tserver_addrs_.size()) {
return Status::ServiceUnavailable("unable to gather any tablet server
metadata");
}
-
- // Now that we've assembled all the metadata, we can write to a syscatalog
table.
- RETURN_NOT_OK(WriteSysCatalog());
-
+ if (!filter_tables.empty()) {
+ RETURN_NOT_OK(UpsertSysCatalog());
+ } else {
+ // Now that we've assembled all the metadata, we can write to a syscatalog
table.
+ RETURN_NOT_OK(WriteSysCatalog());
+ }
state_ = State::DONE;
return Status::OK();
}
@@ -188,7 +252,6 @@ void MasterRebuilder::CreateTable(const
ListTabletsResponsePB::StatusAndSchemaPB
SysTablesEntryPB* metadata = &table->mutable_metadata()->mutable_dirty()->pb;
const string& table_name = replica.tablet_status().table_name();
metadata->set_name(table_name);
-
if (!replica.has_schema_version()) {
metadata->set_version(FLAGS_default_schema_version);
} else {
@@ -375,25 +438,68 @@ Status MasterRebuilder::WriteSysCatalog() {
RETURN_NOT_OK(s);
// Table-by-table, organize the metadata and write it to the syscatalog.
- vector<scoped_refptr<TabletInfo>> tablets;
- const auto kLeaderTimeout = MonoDelta::FromSeconds(10);
-
RETURN_NOT_OK(sys_catalog.tablet_replica()->consensus()->WaitUntilLeader(kLeaderTimeout));
+ RETURN_NOT_OK(DoWrite(&sys_catalog, tables_by_name_,
+ SysCatalogTable::SysCatalogOperation::ADD));
+ return Status::OK();
+}
+
+Status MasterRebuilder::UpsertSysCatalog() {
+ MasterOptions opts;
+ Master master(opts);
+
+ RETURN_NOT_OK(master.Init());
+ SysCatalogTable sys_catalog(&master, &NoOpCb);
+ RETURN_NOT_OK(sys_catalog.Load(master.fs_manager()));
+ SCOPED_CLEANUP({
+ sys_catalog.Shutdown();
+ master.Shutdown();
+ });
+ // Get all table in syscatalog.
+ TableInfoLoader table_info_loader;
+ sys_catalog.VisitTables(&table_info_loader);
+ map<string, scoped_refptr<master::TableInfo>> table_by_name_in_syscatalog;
+ for (const auto& table : table_info_loader.tables) {
+ table->metadata().ReadLock();
+ InsertOrDie(&table_by_name_in_syscatalog, table->metadata().state().name(),
+ table);
+ table->metadata().ReadUnlock();
+ }
+ // Get all tablets in master.
+ TabletInfoLoader tablet_info_loader;
+ sys_catalog.VisitTablets(&tablet_info_loader);
+
+ string new_table_name;
+ map<string, scoped_refptr<master::TableInfo>> table_by_name_to_delete;
+ map<string, vector<scoped_refptr<TabletInfo>>> tablet_by_name_to_delete;
+ map<string, scoped_refptr<master::TableInfo>> table_by_name_to_add;
for (const auto& table_entry : tables_by_name_) {
- const auto& table = table_entry.second;
- table->GetAllTablets(&tablets);
- TableMetadataLock l_table(table.get(), LockMode::WRITE);
- TabletMetadataGroupLock l_tablets(LockMode::RELEASED);
- l_tablets.AddMutableInfos(tablets);
- l_tablets.Lock(LockMode::WRITE);
- SysCatalogTable::Actions actions;
- actions.table_to_add = table;
- actions.tablets_to_add = tablets;
- RETURN_NOT_OK_PREPEND(sys_catalog.Write(actions),
- Substitute("unable to write metadata for table $0 to
sys_catalog",
- table_entry.first));
+ table_entry.second->metadata().ReadLock();
+ new_table_name = table_entry.second->metadata().state().name();
+ table_entry.second->metadata().ReadUnlock();
+ if (ContainsKey(table_by_name_in_syscatalog, new_table_name)) {
+ InsertOrDie(&table_by_name_to_delete, new_table_name,
+ table_by_name_in_syscatalog[new_table_name]);
+ // Get all tablets for this table.
+ vector<scoped_refptr<TabletInfo>> tablets;
+ for (const auto& tablet : tablet_info_loader.tablets) {
+ tablet->metadata().ReadLock();
+ if (tablet->metadata().state().pb.table_id() ==
+ table_by_name_in_syscatalog[new_table_name]->id())
+ tablets.push_back(tablet);
+ tablet->metadata().ReadUnlock();
+ }
+ InsertOrDie(&tablet_by_name_to_delete, new_table_name, tablets);
+ }
+ InsertOrDie(&table_by_name_to_add, table_entry.first, table_entry.second);
}
+
+ // If a table found in master, delete it then add it.
+ RETURN_NOT_OK(DoWrite(&sys_catalog, table_by_name_to_delete,
+ SysCatalogTable::SysCatalogOperation::DELETE,
+ &tablet_by_name_to_delete));
+ RETURN_NOT_OK(DoWrite(&sys_catalog, table_by_name_to_add,
+ SysCatalogTable::SysCatalogOperation::ADD));
return Status::OK();
}
-
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/master_rebuilder.h
b/src/kudu/tools/master_rebuilder.h
index d8d5110a8..b66b852ee 100644
--- a/src/kudu/tools/master_rebuilder.h
+++ b/src/kudu/tools/master_rebuilder.h
@@ -30,8 +30,8 @@
#include "kudu/util/status.h"
namespace kudu {
-namespace tools {
+namespace tools {
// Object for accumulating information about the rebuilding process.
struct RebuildReport {
// List of (address, status) for each tablet server contacted.
@@ -110,6 +110,9 @@ class MasterRebuilder {
// Write the syscatalog table based on the collated tablet server metadata.
Status WriteSysCatalog();
+ // Update or write the syscatalog table based on the collated tablet server
metadata.
+ Status UpsertSysCatalog();
+
State state_;
// Addresses of the tablet servers used for the reconstruction.
@@ -128,6 +131,5 @@ class MasterRebuilder {
DISALLOW_COPY_AND_ASSIGN(MasterRebuilder);
};
-
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/tool_action_master.cc
b/src/kudu/tools/tool_action_master.cc
index a68251f74..088e31217 100644
--- a/src/kudu/tools/tool_action_master.cc
+++ b/src/kudu/tools/tool_action_master.cc
@@ -72,6 +72,7 @@ DECLARE_int64(timeout_ms);
DECLARE_string(columns);
DECLARE_string(fs_wal_dir);
DECLARE_string(fs_data_dirs);
+DECLARE_string(tables);
DEFINE_string(master_uuid, "", "Permanent UUID of the master. Only needed to
disambiguate in case "
"of multiple masters with same RPC address");
@@ -906,7 +907,7 @@ unique_ptr<Mode> BuildMasterMode() {
}
{
- const char* rebuild_extra_description = "Attempts to create on-disk
metadata "
+ const char* rebuild_extra_description = "Attempts to create or update
on-disk metadata "
"that can be used by a non-replicated master to recover a Kudu cluster
"
"that has permanently lost its masters. It has a number of
limitations:\n\n"
" - Security metadata like cryptographic keys are not rebuilt. Tablet
servers "
@@ -927,6 +928,9 @@ unique_ptr<Mode> BuildMasterMode() {
"a very large number.\n\n"
" - Table metadata like comments, owners, and configurations are not
stored on "
"tablet servers and are thus not restored.\n\n"
+ " - Without '--tables', the tool will build a brand new syscatalog
table based on "
+ "tablet data on tablet server metadata.\n\n"
+ " - With '--tables', the tool will update specific tables in current
syscatalog.\n\n"
"WARNING: This tool is potentially unsafe. Only use it when there is
no "
"possibility of recovering the original masters, and you know what you
"
"are doing.\n";
@@ -940,6 +944,7 @@ unique_ptr<Mode> BuildMasterMode() {
.AddOptionalParameter("fs_data_dirs")
.AddOptionalParameter("fs_metadata_dir")
.AddOptionalParameter("fs_wal_dir")
+ .AddOptionalParameter("tables")
.Build();
builder.AddAction(std::move(unsafe_rebuild));
}
