This is an automated email from the ASF dual-hosted git repository. alexey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 3ccc434ea6dcbef15a12bd1734c8e66b69a611c5 Author: yejiabao <[email protected]> AuthorDate: Sun Oct 10 22:32:43 2021 +0800 [tool] Add a '-list_statistics' flag to the 'kudu table list'. '-list_statistics' flag can help to list table's statistics such as tablet number, replica number, and record number Change-Id: I04e9fba724648cd2b774d031e884c5a25cc4084f Reviewed-on: http://gerrit.cloudera.org:8080/17912 Tested-by: Kudu Jenkins Reviewed-by: Bankim Bhavsar <[email protected]> --- src/kudu/client/client.cc | 30 ++++++++++++++++++++++++++++++ src/kudu/client/client.h | 18 ++++++++++++++++++ src/kudu/master/catalog_manager.cc | 7 +++++++ src/kudu/master/master.proto | 3 +++ src/kudu/tools/kudu-tool-test.cc | 29 ++++++++++++++++++++++++++++- src/kudu/tools/tool_action_table.cc | 19 +++++++++++++++---- 6 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc index 19ed0e9..ba1bbf9 100644 --- a/src/kudu/client/client.cc +++ b/src/kudu/client/client.cc @@ -592,6 +592,36 @@ Status KuduClient::ListTables(vector<string>* tables, return Status::OK(); } +Status KuduClient::ListTables(std::vector<ListTableInfo>* list_table_infos, + const std::string& filter) { + ListTablesRequestPB req; + ListTablesResponsePB resp; + + if (!filter.empty()) { + req.set_name_filter(filter); + } + MonoTime deadline = MonoTime::Now() + default_admin_operation_timeout(); + Synchronizer sync; + AsyncLeaderMasterRpc<ListTablesRequestPB, ListTablesResponsePB> rpc( + deadline, this, BackoffType::EXPONENTIAL, req, &resp, + &MasterServiceProxy::ListTablesAsync, "ListTables", + sync.AsStatusCallback(), {}); + rpc.SendRpc(); + RETURN_NOT_OK(sync.Wait()); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + for (const auto& table : resp.tables()) { + ListTableInfo list_table_info; + list_table_info.table_name = table.name(); + list_table_info.live_row_count = table.has_live_row_count() ? table.live_row_count() : 0; + list_table_info.num_tablets = table.has_num_tablets() ? table.num_tablets() : 0; + list_table_info.num_replicas = table.has_num_replicas() ? table.num_replicas() : 0; + list_table_infos->emplace_back(std::move(list_table_info)); + } + return Status::OK(); +} + Status KuduClient::TableExists(const string& table_name, bool* exists) { auto s = GetTableSchema(table_name, nullptr); if (s.ok()) { diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h index 6d72f98..243865d 100644 --- a/src/kudu/client/client.h +++ b/src/kudu/client/client.h @@ -564,6 +564,7 @@ class KUDU_EXPORT KuduTransaction : private: friend class KuduTransaction; class KUDU_NO_EXPORT Data; + Data* data_; // Owned. DISALLOW_COPY_AND_ASSIGN(SerializationOptions); @@ -746,6 +747,22 @@ class KUDU_EXPORT KuduClient : public sp::enable_shared_from_this<KuduClient> { Status ListTables(std::vector<std::string>* tables, const std::string& filter = ""); + struct KUDU_EXPORT ListTableInfo { + std::string table_name; + uint64_t live_row_count; + int num_tablets; + int num_replicas; + }; + /// List only those tables whose names pass a substring match on @c filter. + /// + /// @param [out] list_table_infos + /// The placeholder for the result. Appended only on success. + /// @param [in] filter + /// Substring filter to use; empty sub-string filter matches all tables. + /// @return Status object for the operation. + Status ListTables(std::vector<ListTableInfo>* list_table_infos, + const std::string& filter = ""); + /// Check if the table given by 'table_name' exists. /// /// @param [in] table_name @@ -1265,6 +1282,7 @@ class KUDU_EXPORT KuduTableCreator { int32_t seed = 0); private: class KUDU_NO_EXPORT Data; + friend class KuduTableCreator; // Owned. diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc index 12c8873..36b6775 100644 --- a/src/kudu/master/catalog_manager.cc +++ b/src/kudu/master/catalog_manager.cc @@ -3485,15 +3485,22 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, ListTablesResponsePB::TableInfo* table = resp->add_tables(); table->set_id(table_info->id()); table->set_name(table_name); + table->set_live_row_count(table_info->GetMetrics()->live_row_count->value()); + table->set_num_tablets(table_info->num_tablets()); + table->set_num_replicas(ltm.data().pb.num_replicas()); } } else { // Otherwise, pass all tables through. for (const auto& name_and_table_info : table_info_by_name) { const auto& table_name = name_and_table_info.first; const auto& table_info = name_and_table_info.second; + TableMetadataLock ltm(table_info.get(), LockMode::READ); ListTablesResponsePB::TableInfo* table = resp->add_tables(); table->set_id(table_info->id()); table->set_name(table_name); + table->set_live_row_count(table_info->GetMetrics()->live_row_count->value()); + table->set_num_tablets(table_info->num_tablets()); + table->set_num_replicas(ltm.data().pb.num_replicas()); } } return Status::OK(); diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto index a6e3f7a..b9ded52 100644 --- a/src/kudu/master/master.proto +++ b/src/kudu/master/master.proto @@ -594,6 +594,9 @@ message ListTablesResponsePB { message TableInfo { required bytes id = 1; required string name = 2; + optional uint64 live_row_count = 3; + optional int32 num_tablets = 4; + optional int32 num_replicas = 5; } repeated TableInfo tables = 2; diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 090ff11..a0ba47f 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -4090,13 +4090,14 @@ TEST_F(ToolTest, TestListTables) { // Create some tables. const int kNumTables = 10; vector<string> table_names; + const int kReplicaNum = 1; for (int i = 0; i < kNumTables; ++i) { string table_name = Substitute("kudu.table_$0", i); table_names.push_back(table_name); TestWorkload workload(cluster_.get()); workload.set_table_name(table_name); - workload.set_num_replicas(1); + workload.set_num_replicas(kReplicaNum); workload.Setup(); } std::sort(table_names.begin(), table_names.end()); @@ -4156,10 +4157,36 @@ TEST_F(ToolTest, TestListTables) { } }; + const auto& ProcessTablesStatistics = [&] (const int num) { + ASSERT_GE(num, 1); + ASSERT_LE(num, kNumTables); + + vector<string> expected; + expected.reserve(num); + for (int i = 0; i < num; i++) { + expected.push_back(Substitute("$0 Num_Tablets:1 Num_Replicas:$1 Live_Row_Count:0", + table_names[i], kReplicaNum)); + } + vector<string> expected_table; + expected_table.insert(expected_table.end(), table_names.begin(), table_names.begin() + num); + + string filter = ""; + if (kNumTables != num) { + filter = Substitute("-tables=$0", JoinStrings(expected_table, ",")); + } + vector<string> lines; + NO_FATALS(RunActionStdoutLines( + Substitute("table list $0 $1 -list_statistics", master_addr, filter), &lines)); + + std::sort(lines.begin(), lines.end()); + ASSERT_EQ(expected, lines); + }; + // List the tables and tablets. for (int i = 1; i <= kNumTables; ++i) { ProcessTables(i); ProcessTablets(i); + ProcessTablesStatistics(i); } } diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc index 32bfd63..77e2f75 100644 --- a/src/kudu/tools/tool_action_table.cc +++ b/src/kudu/tools/tool_action_table.cc @@ -103,6 +103,9 @@ DEFINE_string(dst_table, "", "If the empty string, use the same name as the source table."); DEFINE_bool(list_tablets, false, "Include tablet and replica UUIDs in the output"); +DEFINE_bool(list_statistics, false, + "Include statistics such as number of tablets, replicas" + "and live row count in the output"); DEFINE_bool(modify_external_catalogs, true, "Whether to modify external catalogs, such as the Hive Metastore, " "when renaming or dropping a table."); @@ -139,13 +142,20 @@ class TableLister { RETURN_NOT_OK(CreateKuduClient(master_addresses, &client, true /* can_see_all_replicas */)); - vector<string> table_names; - RETURN_NOT_OK(client->ListTables(&table_names)); + std::vector<kudu::client::KuduClient::ListTableInfo> list_table_infos; + RETURN_NOT_OK(client->ListTables(&list_table_infos)); vector<string> table_filters = Split(FLAGS_tables, ",", strings::SkipEmpty()); - for (const auto& tname : table_names) { + for (const auto& list_table_info : list_table_infos) { + const auto& tname = list_table_info.table_name; if (!MatchesAnyPattern(table_filters, tname)) continue; - cout << tname << endl; + if (FLAGS_list_statistics) { + cout << tname << " " << "Num_Tablets:" << list_table_info.num_tablets + << " Num_Replicas:" << list_table_info.num_replicas + << " Live_Row_Count:" << list_table_info.live_row_count << endl; + } else { + cout << tname << endl; + } if (!FLAGS_list_tablets) { continue; } @@ -1317,6 +1327,7 @@ unique_ptr<Mode> BuildTableMode() { .Description("List tables") .AddOptionalParameter("tables") .AddOptionalParameter("list_tablets") + .AddOptionalParameter("list_statistics") .Build(); unique_ptr<Action> locate_row =
