This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 1f89113a6 [tools] Run range aware cluster rebalance with multiple
tables
1f89113a6 is described below
commit 1f89113a618a84d2610410600c012c6180332adc
Author: 0xderek <[email protected]>
AuthorDate: Thu Dec 19 10:25:50 2024 +0800
[tools] Run range aware cluster rebalance with multiple tables
Previously when rebalancing cluster with kudu command line tool, the
`--enable_range_rebalancing` flag requires the `--tables` flag with exactly
one
table specified.
This commit removes this restriction, allowing multiple tables to be passed
to
the `--tables` flag when range rebalancing is enabled. Among them, range
partitioned tables will be rebalanced with ranges considered, while others
are
rebalanced as usual. If the `--tables` flag is not set, all tables in the
cluster will be rebalanced.
This commit also optimizes the replica distribution details when range
rabalancing is applied to multiple tables with mixed partition schemas.
Change-Id: I7f6995496e6c28e6a3a8fa4b6f0e8efd05335267
Reviewed-on: http://gerrit.cloudera.org:8080/22239
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
---
src/kudu/rebalance/cluster_status.h | 1 +
src/kudu/rebalance/rebalance-test.cc | 4 +-
src/kudu/rebalance/rebalancer.cc | 3 +-
src/kudu/tools/ksck.cc | 2 +
src/kudu/tools/rebalancer_tool-test.cc | 108 +++++++++++++++++++++++++++++++--
src/kudu/tools/rebalancer_tool.cc | 30 ++++++---
src/kudu/tools/tool_action_cluster.cc | 6 --
7 files changed, 133 insertions(+), 21 deletions(-)
diff --git a/src/kudu/rebalance/cluster_status.h
b/src/kudu/rebalance/cluster_status.h
index 749c88e51..9836c45f0 100644
--- a/src/kudu/rebalance/cluster_status.h
+++ b/src/kudu/rebalance/cluster_status.h
@@ -157,6 +157,7 @@ struct TableSummary {
int underreplicated_tablets = 0;
int consensus_mismatch_tablets = 0;
int unavailable_tablets = 0;
+ bool is_range_partitioned = false;
int TotalTablets() const {
return healthy_tablets + recovering_tablets + underreplicated_tablets +
diff --git a/src/kudu/rebalance/rebalance-test.cc
b/src/kudu/rebalance/rebalance-test.cc
index 94706d4a2..38a999059 100644
--- a/src/kudu/rebalance/rebalance-test.cc
+++ b/src/kudu/rebalance/rebalance-test.cc
@@ -702,7 +702,9 @@ TEST_F(KsckResultsToClusterBalanceInfoTest,
RangeRebalancingInfo) {
},
};
- NO_FATALS(RunTest(Rebalancer::Config(), test_configs));
+ Rebalancer::Config config;
+ config.enable_range_rebalancing = true;
+ NO_FATALS(RunTest(config, test_configs));
}
} // namespace rebalance
diff --git a/src/kudu/rebalance/rebalancer.cc b/src/kudu/rebalance/rebalancer.cc
index 738512174..092f7b4f0 100644
--- a/src/kudu/rebalance/rebalancer.cc
+++ b/src/kudu/rebalance/rebalancer.cc
@@ -420,7 +420,8 @@ Status Rebalancer::BuildClusterInfo(const ClusterRawInfo&
raw_info,
}
auto table_ins = table_replicas_info.emplace(
- TableIdAndTag{tablet.table_id, tablet.range_key_begin},
+ TableIdAndTag{tablet.table_id,
+ config_.enable_range_rebalancing ?
tablet.range_key_begin : ""},
TableReplicasAtServer());
TableReplicasAtServer& replicas_at_server = table_ins.first->second;
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index 522eec7d4..b9cf8e0bc 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -845,6 +845,8 @@ bool Ksck::VerifyTable(const shared_ptr<KsckTable>& table,
vector<TableSummary>*
ts.id = table->id();
ts.name = table->name();
ts.replication_factor = table->num_replicas();
+ const auto& tablet = table->tablets().front();
+ ts.is_range_partitioned = !tablet->partition().begin().range_key().empty();
VLOG(1) << Substitute("Verifying $0 tablet(s) for table $1 configured with
num_replicas = $2",
table->tablets().size(), table->name(),
table->num_replicas());
for (const auto& tablet : table->tablets()) {
diff --git a/src/kudu/tools/rebalancer_tool-test.cc
b/src/kudu/tools/rebalancer_tool-test.cc
index 597ef71b7..938a07852 100644
--- a/src/kudu/tools/rebalancer_tool-test.cc
+++ b/src/kudu/tools/rebalancer_tool-test.cc
@@ -2307,6 +2307,12 @@ class TableRangeRebalancingTest : public RebalancingTest
{
int num_range_partitions_for_test_tables() const override {
return 2;
}
+
+ // There should be 8 tablet replicas for a particular range per tablet
server.
+ static constexpr char kFirstRangeReferencePattern[] =
+ "Range start key: '00000000'\n.+\n.+\n .+ 8\n .+ 8\n .+ 8\n";
+ static constexpr char kSecondRangeReferencePattern[] =
+ "Range start key: 'ff80000000'\n.+\n.+\n .+ 8\n .+ 8\n .+ 8\n";
};
TEST_F(TableRangeRebalancingTest, Basic) {
@@ -2328,12 +2334,6 @@ TEST_F(TableRangeRebalancingTest, Basic) {
Maximum | 0
Average | 0)***";
- // There should be 8 tablet replicas for a particular range per tablet
server.
- constexpr const char kFirstRangeReferencePattern[] =
- "Range start key: '00000000'\n.+\n.+\n .+ 8\n .+ 8\n .+ 8\n";
- constexpr const char kSecondRangeReferencePattern[] =
- "Range start key: 'ff80000000'\n.+\n.+\n .+ 8\n .+ 8\n .+ 8\n";
-
vector<string> table_names;
NO_FATALS(Prepare({}, {}, {}, kEmptySet, &table_names));
ASSERT_EQ(1, table_names.size());
@@ -2373,5 +2373,101 @@ TEST_F(TableRangeRebalancingTest, Basic) {
}
}
+TEST_F(TableRangeRebalancingTest, ClusterWideRangeRebalancing) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ const auto schema_flag =
+ Substitute("--raft_prepare_replacement_before_eviction=$0",
is_343_scheme());
+ FLAGS_num_tablet_servers = num_tservers_;
+ FLAGS_num_replicas = rep_factor_;
+ NO_FATALS(
+ BuildAndStart({schema_flag},
+ {schema_flag,
+ "--enable_range_replica_placement=false",
+ Substitute("--tserver_unresponsive_timeout_ms=$0",
tserver_unresponsive_ms_)},
+ {},
+ false));
+
+ string range_partitioned_table_name;
+ string non_range_partitioned_table_name;
+ vector<string> table_names;
+
+ ASSERT_OK(CreateUnbalancedTables(cluster_.get(),
+ client_.get(),
+ schema_,
+ "range_partitioned_table_$0",
+ 1,
+ rep_factor_,
+ rep_factor_ + 1,
+ num_tservers_,
+ tserver_unresponsive_ms_,
+ num_hash_partitions_for_test_tables(),
+ num_range_partitions_for_test_tables(),
+ &table_names));
+ ASSERT_EQ(1, table_names.size());
+ range_partitioned_table_name = table_names[0];
+
+ ASSERT_OK(CreateUnbalancedTables(cluster_.get(),
+ client_.get(),
+ schema_,
+ "non_range_partitioned_table_$0",
+ 1,
+ rep_factor_,
+ rep_factor_ + 1,
+ num_tservers_,
+ tserver_unresponsive_ms_,
+ num_hash_partitions_for_test_tables(),
+ 0,
+ &table_names));
+ ASSERT_EQ(2, table_names.size());
+ non_range_partitioned_table_name = table_names[1];
+
+ {
+ const vector<string> tool_args = {"cluster",
+ "rebalance",
+
cluster_->master()->bound_rpc_addr().ToString(),
+ "--enable_range_rebalancing"};
+ string out;
+ string err;
+ const Status s = RunKuduTool(tool_args, &out, &err);
+ ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+ ASSERT_STR_CONTAINS(out, "rebalancing is complete: cluster is balanced")
<< "stderr: " << err;
+ }
+
+ {
+ const vector<string> report_tool_args = {"cluster",
+ "rebalance",
+
cluster_->master()->bound_rpc_addr().ToString(),
+ "--enable_range_rebalancing",
+ "--report_only",
+
"--output_replica_distribution_details"};
+ string out;
+ string err;
+ const Status s = RunKuduTool(report_tool_args, &out, &err);
+ ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+
+ const string rangePartitionedTableStats =
+ string("Per-range replica distribution details for tables\n\n") +
+ "Table: .+" + range_partitioned_table_name + ".+\n\n" +
+ "Number of tablet replicas at servers for each range\n" +
+ " Max Skew | Total Count | Range Start Key\n" +
+ "----------+-------------+-----------------\n" +
+ " 0 | 24 | 00000000\n" +
+ " 0 | 24 | ff80000000";
+
+ const auto nonRangePartitionedTableStats =
+ string("Per-table replica distribution details for non range
partitioned tables:\n") +
+ " Table Id | Replica Count | Replica Skew |
Table Name\n" +
+
"----------------------------------+---------------+--------------+------------------------"
+ "-------\n" +
+ " .+ | 24 | 0 | " +
non_range_partitioned_table_name;
+
+ ASSERT_STR_MATCHES(out, rangePartitionedTableStats);
+ ASSERT_STR_MATCHES(out, kFirstRangeReferencePattern);
+ ASSERT_STR_MATCHES(out, kSecondRangeReferencePattern);
+ ASSERT_STR_MATCHES(out, nonRangePartitionedTableStats);
+ }
+}
+
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/rebalancer_tool.cc
b/src/kudu/tools/rebalancer_tool.cc
index 5c95e19ba..0af0324da 100644
--- a/src/kudu/tools/rebalancer_tool.cc
+++ b/src/kudu/tools/rebalancer_tool.cc
@@ -586,6 +586,9 @@ Status RebalancerTool::PrintLocationBalanceStats(const
string& location,
for (const auto& [_, balance_info] : table_skew_info) {
const auto& table_id = balance_info.table_id;
const auto& tag = balance_info.tag;
+ if (!table_info[table_id]->is_range_partitioned) {
+ continue;
+ }
auto it = range_dist_stats.emplace(
std::make_pair(table_id, tag), map<string, size_t>{});
const auto& server_info = balance_info.servers_by_replica_count;
@@ -616,16 +619,18 @@ Status RebalancerTool::PrintLocationBalanceStats(const
string& location,
}
string prev_table_id;
- for (const auto& [table_info, per_server_stats] : range_dist_stats) {
- const auto& table_id = table_info.first;
- const auto& table_range = table_info.second;
+ for (const auto& [table_id_and_tag, per_server_stats] :
range_dist_stats) {
+ const auto& table_id = table_id_and_tag.first;
+ const auto& table_range = table_id_and_tag.second;
if (prev_table_id != table_id) {
prev_table_id = table_id;
- out << endl << "Table: " << table_id << endl << endl;
+ out << endl
+ << Substitute("Table: $0 ($1)", table_id,
table_info[table_id]->name) << endl
+ << endl;
out << "Number of tablet replicas at servers for each range" <<
endl;
DataTable range_skew_summary_table(
{ "Max Skew", "Total Count", "Range Start Key" });
- const auto it_begin = range_skew_stats.find(table_info);
+ const auto it_begin = range_skew_stats.find(table_id_and_tag);
for (auto it = it_begin; it != range_skew_stats.end(); ++it) {
const auto& cur_table_id = it->first.first;
if (cur_table_id != table_id) {
@@ -650,12 +655,23 @@ Status RebalancerTool::PrintLocationBalanceStats(const
string& location,
RETURN_NOT_OK(skew_table.PrintTo(out));
out << endl;
}
- } else {
- out << "Per-table replica distribution details:" << endl;
+ }
+
+ const auto has_non_range_partitioned_table =
+ std::any_of(table_info.begin(), table_info.end(), [&](const auto&
info) {
+ return !info.second->is_range_partitioned;
+ });
+ if (!config_.enable_range_rebalancing ||
has_non_range_partitioned_table) {
+ out << "Per-table replica distribution details for "
+ << (config_.enable_range_rebalancing ? "non range partitioned
tables:" : "tables:")
+ << endl;
DataTable skew_table(
{ "Table Id", "Replica Count", "Replica Skew", "Table Name" });
for (const auto& [skew, balance_info] : table_skew_info) {
const auto& table_id = balance_info.table_id;
+ if (config_.enable_range_rebalancing &&
table_info[table_id]->is_range_partitioned) {
+ continue;
+ }
const auto it = table_info.find(table_id);
const auto* table_summary =
(it == table_info.end()) ? nullptr : it->second;
diff --git a/src/kudu/tools/tool_action_cluster.cc
b/src/kudu/tools/tool_action_cluster.cc
index b422ddd56..9fa10ce11 100644
--- a/src/kudu/tools/tool_action_cluster.cc
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -326,12 +326,6 @@ Status RunRebalance(const RunnerContext& context) {
const vector<string> table_filters =
Split(FLAGS_tables, ",", strings::SkipEmpty());
- if (FLAGS_enable_range_rebalancing && table_filters.size() != 1) {
- return Status::NotSupported(
- "range rebalancing is currently implemented for a single table only: "
- "use '--tables' to specify a table for range rebalancing");
- }
-
// Evaluate --move_single_replicas flag: decide whether enable to disable
// moving of single-replica tablets based on the reported version of the
// Kudu components.
