This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 7f7e2cf46 KUDU-3403 Enable kudu CLI to accept specific leader to step
down
7f7e2cf46 is described below
commit 7f7e2cf4611e9fb6fe07a57df8762f1ce2e71b8f
Author: Ashwani Raina <[email protected]>
AuthorDate: Tue Sep 13 05:26:26 2022 -0700
KUDU-3403 Enable kudu CLI to accept specific leader to step down
Accept UUID as an argument to identify the leader
and deduce other information like host, port, etc.
This can be useful in scenario when master is out of sync and
user wants to choose a specific leader to step down instead of
relying on information about leader in the tablet configuration
known to master.
Change-Id: I40569faa40a8173c51504c7567aa84a6ae1fb64a
Reviewed-on: http://gerrit.cloudera.org:8080/19024
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
---
src/kudu/tools/kudu-admin-test.cc | 75 ++++++++++++++++++++++++++++++++++++
src/kudu/tools/tool_action_common.cc | 1 +
src/kudu/tools/tool_action_common.h | 2 +
src/kudu/tools/tool_action_tablet.cc | 52 ++++++++++++++++---------
src/kudu/tools/tool_replica_util.cc | 22 ++++++++++-
src/kudu/tools/tool_replica_util.h | 11 ++++++
6 files changed, 144 insertions(+), 19 deletions(-)
diff --git a/src/kudu/tools/kudu-admin-test.cc
b/src/kudu/tools/kudu-admin-test.cc
index f32bba194..3e12c7e0a 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -1357,6 +1357,81 @@ TEST_F(AdminCliTest, TestGracefulLeaderStepDown) {
});
}
+// Test current_leader_uuid flag used to transfer leadership from current node
+// 1. Elect a leader and transfer leadership to a new node.
+// 2. Input new node uuid as an argument to flag current_leader_uuid.
+// 3. Transfer the leadership from current leader to another node.
+// 4. Verify the new leader chosen is one of the two non-leader nodes.
+TEST_F(AdminCliTest, TestGracefulSpecificLeaderStepDown) {
+ const MonoDelta kTimeout = MonoDelta::FromSeconds(10);
+ const vector<string> kMasterFlags = {
+ "--catalog_manager_wait_for_new_tablets_to_elect_leader=false",
+ };
+
+ // For deterministic control over manual leader selection
+ const vector<string> kTserverFlags = {
+ "--enable_leader_failure_detection=false",
+ };
+ FLAGS_num_tablet_servers = 3;
+ FLAGS_num_replicas = 3;
+ NO_FATALS(BuildAndStart(kTserverFlags, kMasterFlags));
+
+ // Wait for the tablet to be running.
+ vector<TServerDetails*> tservers;
+ AppendValuesFromMap(tablet_servers_, &tservers);
+ ASSERT_EQ(FLAGS_num_tablet_servers, tservers.size());
+ for (auto& ts : tservers) {
+ ASSERT_OK(WaitUntilTabletRunning(ts, tablet_id_, kTimeout));
+ }
+
+ // Elect the leader and wait for the tservers and master to see the leader.
+ const auto* leader = tservers[0];
+ ASSERT_OK(StartElection(leader, tablet_id_, kTimeout));
+ ASSERT_OK(WaitForServersToAgree(kTimeout, tablet_servers_,
+ tablet_id_, /*minimum_index=*/1));
+ TServerDetails* master_observed_leader;
+ ASSERT_OK(GetLeaderReplicaWithRetries(tablet_id_, &master_observed_leader));
+ ASSERT_EQ(leader->uuid(), master_observed_leader->uuid());
+
+ const auto leader_uuid = leader->uuid();
+ string stderr;
+
+ // Ask the leader to transfer leadership and specify the current leader uuid.
+ // Use the leader elected above as current leader input, instead of relying
on
+ // master configuration. This is to check whether user specified current
+ // leader uuid is ok to be used as input parameter for new leader election.
+ // Generally, we rely on master configuration to find that out. But for cases
+ // where master configuration is out of sync, user input parameter for
+ // current leader should work just as well.
+ Status s = RunKuduTool({
+ "tablet",
+ "leader_step_down",
+ Substitute("--current_leader_uuid=$0", leader_uuid),
+ cluster_->master()->bound_rpc_addr().ToString(),
+ tablet_id_
+ }, nullptr, &stderr);
+ ASSERT_OK(s);
+
+ // Eventually, a replica at other node should become a leader.
+ const std::unordered_set<string> possible_new_leaders = {
tservers[1]->uuid(),
+
tservers[2]->uuid() };
+ ASSERT_EVENTUALLY([&]() {
+ ASSERT_OK(GetLeaderReplicaWithRetries(tablet_id_,
&master_observed_leader));
+ ASSERT_TRUE(ContainsKey(possible_new_leaders,
master_observed_leader->uuid()));
+ });
+
+ // Negative case: When current_leader_uuid belongs to a tablet server
+ // where a follower (and not a leader) replica is running.
+ s = RunKuduTool({
+ "tablet",
+ "leader_step_down",
+ Substitute("--current_leader_uuid=$0", leader_uuid),
+ cluster_->master()->bound_rpc_addr().ToString(),
+ tablet_id_
+ }, nullptr, &stderr);
+ ASSERT_TRUE(s.IsRuntimeError()) << s.ToString();
+}
+
// Leader should reject requests to transfer leadership to a non-member of the
// config.
TEST_F(AdminCliTest, TestLeaderTransferToEvictedPeer) {
diff --git a/src/kudu/tools/tool_action_common.cc
b/src/kudu/tools/tool_action_common.cc
index b7dd58b76..ca28e04a2 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -262,6 +262,7 @@ namespace kudu {
namespace tools {
const char* const kMasterAddressesArg = "master_addresses";
+const char* const kCurrentLeaderUUIDArg = "current_leader_uuid";
const char* const kMasterAddressesArgDesc = "Either comma-separated list of
Kudu "
"master addresses where each address is of form 'hostname:port', or a
cluster name if it has "
"been configured in ${KUDU_CONFIG}/kudurc";
diff --git a/src/kudu/tools/tool_action_common.h
b/src/kudu/tools/tool_action_common.h
index ada659b92..c6e5162d3 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -80,6 +80,8 @@ extern const char* const kMasterAddressDesc;
extern const char* const kTServerAddressArg;
extern const char* const kTServerAddressDesc;
+extern const char* const kCurrentLeaderUUIDArg;
+
// Builder for actions involving RPC communications, either with a whole Kudu
// cluster or a particular Kudu RPC server.
class RpcActionBuilder : public ActionBuilder {
diff --git a/src/kudu/tools/tool_action_tablet.cc
b/src/kudu/tools/tool_action_tablet.cc
index a900f2abc..c4351b33c 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -21,13 +21,14 @@
#include <memory>
#include <optional>
#include <string>
+#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
#include <gflags/gflags.h>
-#include "kudu/client/client.h"
+#include "kudu/client/client.h" // IWYU pragma: keep
#include "kudu/client/shared_ptr.h" // IWYU pragma: keep
#include "kudu/common/wire_protocol.h"
#include "kudu/consensus/consensus.pb.h"
@@ -37,7 +38,6 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/master/master.pb.h"
#include "kudu/master/master.proxy.h"
-#include "kudu/rpc/response_callback.h"
#include "kudu/tools/tool_action.h"
#include "kudu/tools/tool_action_common.h"
#include "kudu/tools/tool_replica_util.h"
@@ -59,6 +59,8 @@ DEFINE_string(new_leader_uuid, "",
"transfer will not occur. If blank, the leader chooses its own "
"successor, attempting to transfer leadership as soon as "
"possible. This cannot be set if --abrupt is set.");
+DEFINE_string(current_leader_uuid, "",
+ "UUID of the server that currently hosts leader replica of the
tablet.");
DEFINE_int64(move_copy_timeout_sec, 600,
"Number of seconds to wait for tablet copy to complete when
relocating a tablet");
DEFINE_int64(move_leader_timeout_sec, 30,
@@ -154,37 +156,50 @@ Status LeaderStepDown(const RunnerContext& context) {
return Status::InvalidArgument("cannot specify both --new_leader_uuid and
--abrupt");
}
+ const optional<string> current_leader_uuid =
FLAGS_current_leader_uuid.empty()
+ ? nullopt
+ : make_optional(FLAGS_current_leader_uuid);
client::sp::shared_ptr<KuduClient> client;
RETURN_NOT_OK(CreateKuduClient(master_addresses, &client));
-
string leader_uuid;
HostPort leader_hp;
- bool no_leader = false;
- Status s = GetTabletLeader(client, tablet_id,
- &leader_uuid, &leader_hp, &no_leader);
- if (s.IsNotFound() && no_leader) {
- // If leadership should be transferred to a specific node, exit with an
- // error if there's no leader since we can't orchestrate the transfer.
- if (new_leader_uuid) {
+ Status s;
+
+ if (current_leader_uuid) {
+ s = GetTabletReplicaHostInfo(client, tablet_id, *current_leader_uuid,
+ &leader_hp);
+ if (s.IsNotFound()) {
+ return s.CloneAndPrepend(
+ Substitute("unable to transfer leadership, current leader $0 not
found",
+ *current_leader_uuid));
+ }
+ } else {
+ bool no_leader = false;
+ s = GetTabletLeader(client, tablet_id,
+ &leader_uuid, &leader_hp, &no_leader);
+ if (s.IsNotFound() && no_leader) {
+ // If leadership should be transferred to a specific node, exit with an
+ // error if there's no leader since we can't orchestrate the transfer.
+ if (new_leader_uuid) {
return s.CloneAndPrepend(
Substitute("unable to transfer leadership to $0",
*new_leader_uuid));
+ }
+ // Otherwise, a new election should happen soon, which will achieve
+ // something like what the client wanted, so we'll exit gracefully.
+ cout << s.ToString() << endl;
+ return Status::OK();
}
- // Otherwise, a new election should happen soon, which will achieve
- // something like what the client wanted, so we'll exit gracefully.
- cout << s.ToString() << endl;
- return Status::OK();
}
RETURN_NOT_OK(s);
// If the requested new leader is the leader, the command can short-circuit.
- if (new_leader_uuid && (leader_uuid == *new_leader_uuid)) {
+ if (new_leader_uuid && (leader_uuid == *new_leader_uuid || leader_uuid ==
*current_leader_uuid)) {
cout << Substitute("Requested new leader $0 is already the leader",
leader_uuid) << endl;
return Status::OK();
}
-
- return DoLeaderStepDown(tablet_id, leader_uuid, leader_hp,
- mode, new_leader_uuid,
+ return DoLeaderStepDown(tablet_id, current_leader_uuid ?
*current_leader_uuid : leader_uuid,
+ leader_hp, mode, new_leader_uuid,
client->default_admin_operation_timeout());
}
@@ -315,6 +330,7 @@ unique_ptr<Mode> BuildTabletMode() {
.AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
.AddOptionalParameter("abrupt")
.AddOptionalParameter("new_leader_uuid")
+ .AddOptionalParameter("current_leader_uuid")
.Build();
unique_ptr<Mode> change_config =
diff --git a/src/kudu/tools/tool_replica_util.cc
b/src/kudu/tools/tool_replica_util.cc
index 94ce5a210..f627f3297 100644
--- a/src/kudu/tools/tool_replica_util.cc
+++ b/src/kudu/tools/tool_replica_util.cc
@@ -21,10 +21,10 @@
#include <fstream> // IWYU pragma: keep
#include <memory>
#include <string>
+#include <type_traits>
#include <vector>
#include <glog/logging.h>
-#include <google/protobuf/stubs/port.h>
#include "kudu/client/client.h"
#include "kudu/client/shared_ptr.h" // IWYU pragma: keep
@@ -190,6 +190,26 @@ Status GetTabletLeader(const
client::sp::shared_ptr<KuduClient>& client,
tablet_id));
}
+Status GetTabletReplicaHostInfo(const client::sp::shared_ptr<KuduClient>&
client,
+ const string& tablet_id,
+ const std::string& uuid,
+ HostPort* hp) {
+ KuduTablet* tablet_raw = nullptr;
+ RETURN_NOT_OK(client->GetTablet(tablet_id, &tablet_raw));
+ unique_ptr<KuduTablet> tablet(tablet_raw);
+
+ for (const auto* r : tablet->replicas()) {
+ if (uuid == r->ts().uuid()) {
+ hp->set_host(r->ts().hostname());
+ hp->set_port(r->ts().port());
+ return Status::OK();
+ }
+ }
+
+ return Status::NotFound(Substitute("no replica of tablet $0 is hosted by "
+ "tablet server $1", tablet_id, uuid));
+}
+
// For the target (i.e. newly added replica) we have the following options:
//
// * The tablet copy succeeds and the replica successfully bootstraps and
diff --git a/src/kudu/tools/tool_replica_util.h
b/src/kudu/tools/tool_replica_util.h
index a761ea348..492a01f53 100644
--- a/src/kudu/tools/tool_replica_util.h
+++ b/src/kudu/tools/tool_replica_util.h
@@ -92,6 +92,17 @@ Status GetTabletLeader(
HostPort* leader_hp,
bool* is_no_leader = nullptr);
+// Get host/port information of the tablet server with the specified UUID
'uuid'
+// that hosts a replica of the tablet identified by 'tablet_id'.
+// The 'uuid' is input parameter and 'hp' is output parameter
+// and cannot be null. The function returns Status::NotFound() if no host
+// info is found for given uuid.
+Status GetTabletReplicaHostInfo(
+ const client::sp::shared_ptr<client::KuduClient>& client,
+ const std::string& tablet_id,
+ const std::string& uuid,
+ HostPort* hp);
+
// Whether the replica move operation from 'from_ts_uuid' to 'to_ts_uuid'
// server is complete (i.e. succeeded or failed) for the tablet identified by
// 'tablet_id'. Neither 'is_complete' nor 'completion_status' output parameter
