KUDU-2529: Add a "-tables=<tables>" flag to the "kudu table list".
"-tables=<tables>" flag can help to filter the tables that you want while running "kudu table list <master> -list_tables". Change-Id: I23006a3f3e4f157e6c6c10968b8692e89892d1ce Reviewed-on: http://gerrit.cloudera.org:8080/11360 Reviewed-by: Adar Dembo <a...@cloudera.com> Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4fceeae3 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4fceeae3 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4fceeae3 Branch: refs/heads/master Commit: 4fceeae3f314ef794bd58c47439033941a1aa568 Parents: 6b70c27 Author: helifu <hzhel...@corp.netease.com> Authored: Fri Aug 24 17:14:19 2018 +0800 Committer: Adar Dembo <a...@cloudera.com> Committed: Wed Sep 5 16:00:18 2018 +0000 ---------------------------------------------------------------------- src/kudu/tools/ksck.cc | 14 +--- src/kudu/tools/kudu-tool-test.cc | 127 ++++++++++++++++++++++++++++- src/kudu/tools/tool_action_cluster.cc | 6 +- src/kudu/tools/tool_action_common.cc | 13 +++ src/kudu/tools/tool_action_common.h | 4 + src/kudu/tools/tool_action_table.cc | 7 +- 6 files changed, 152 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/ksck.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc index eab1ac1..c426d38 100644 --- a/src/kudu/tools/ksck.cc +++ b/src/kudu/tools/ksck.cc @@ -39,9 +39,9 @@ #include "kudu/gutil/strings/human_readable.h" #include "kudu/gutil/strings/join.h" #include "kudu/gutil/strings/substitute.h" -#include "kudu/gutil/strings/util.h" #include "kudu/tablet/tablet.pb.h" #include "kudu/tools/color.h" +#include "kudu/tools/tool_action_common.h" #include "kudu/util/atomic.h" #include "kudu/util/blocking_queue.h" #include "kudu/util/countdown_latch.h" @@ -99,18 +99,6 @@ namespace kudu { namespace tools { namespace { -// Return true if 'str' matches any of the patterns in 'patterns', or if -// 'patterns' is empty. -bool MatchesAnyPattern(const vector<string>& patterns, const string& str) { - // Consider no filter a wildcard. - if (patterns.empty()) return true; - - for (const auto& p : patterns) { - if (MatchPattern(str, p)) return true; - } - return false; -} - void BuildKsckConsensusStateForConfigMember(const consensus::ConsensusStatePB& cstate, KsckConsensusState* ksck_cstate) { CHECK(ksck_cstate); http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/kudu-tool-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 28180c4..8669695 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -140,6 +140,8 @@ using kudu::client::KuduClient; using kudu::client::KuduClientBuilder; using kudu::client::KuduSchema; using kudu::client::KuduSchemaBuilder; +using kudu::client::KuduScanToken; +using kudu::client::KuduScanTokenBuilder; using kudu::client::KuduTable; using kudu::client::sp::shared_ptr; using kudu::cluster::ExternalMiniCluster; @@ -176,6 +178,7 @@ using kudu::tserver::MiniTabletServer; using kudu::tserver::WriteRequestPB; using std::back_inserter; using std::copy; +using std::map; using std::make_pair; using std::ostringstream; using std::pair; @@ -581,7 +584,9 @@ TEST_F(ToolTest, TestModeHelp) { { const vector<string> kTableModeRegexes = { "delete.*Delete a table", - "list.*List all tables", + "rename_table.*Rename a table", + "rename_column.*Rename a column", + "list.*List tables", }; NO_FATALS(RunTestHelp("table", kTableModeRegexes)); } @@ -2117,6 +2122,39 @@ TEST_F(ToolTest, TestMasterList) { ASSERT_STR_CONTAINS(out, master->bound_rpc_hostport().ToString()); } +// Operate on Kudu tables: +// (1)delete a table +// (2)rename a table +// (3)rename a column +// (4)list tables +TEST_F(ToolTest, TestDeleteTable) { + NO_FATALS(StartExternalMiniCluster()); + shared_ptr<KuduClient> client; + ASSERT_OK(cluster_->CreateClient(nullptr, &client)); + string master_addr = cluster_->master()->bound_rpc_addr().ToString(); + + const string& kTableName = "kudu.table"; + + // Create a table. + TestWorkload workload(cluster_.get()); + workload.set_table_name(kTableName); + workload.set_num_replicas(1); + workload.Setup(); + + // Check that the table exists. + bool exist = false; + ASSERT_OK(client->TableExists(kTableName, &exist)); + ASSERT_EQ(exist, true); + + // Delete the table. + NO_FATALS(RunActionStdoutNone(Substitute("table delete $0 $1", + master_addr, kTableName))); + + // Check that the table does not exist. + ASSERT_OK(client->TableExists(kTableName, &exist)); + ASSERT_EQ(exist, false); +} + TEST_F(ToolTest, TestRenameTable) { NO_FATALS(StartExternalMiniCluster()); shared_ptr<KuduClient> client; @@ -2183,6 +2221,93 @@ TEST_F(ToolTest, TestRenameColumn) { ASSERT_STR_CONTAINS(table->schema().ToString(), kNewColumnName); } +TEST_F(ToolTest, TestListTables) { + NO_FATALS(StartExternalMiniCluster()); + shared_ptr<KuduClient> client; + ASSERT_OK(cluster_->CreateClient(nullptr, &client)); + string master_addr = cluster_->master()->bound_rpc_addr().ToString(); + + // Replica's format. + string ts_uuid = cluster_->tablet_server(0)->uuid(); + string ts_addr = cluster_->tablet_server(0)->bound_rpc_addr().ToString(); + string expect_replica = Substitute(" L $0 $1", ts_uuid, ts_addr); + + // Create some tables. + const int kNumTables = 10; + vector<string> table_names; + for (int i = 0; i < kNumTables; ++i) { + string table_name = Substitute("kudu.table_$0", i); + table_names.push_back(table_name); + + TestWorkload workload(cluster_.get()); + workload.set_table_name(table_name); + workload.set_num_replicas(1); + workload.Setup(); + } + std::sort(table_names.begin(), table_names.end()); + + const auto& ProcessTables = [&] (const int num) { + ASSERT_GE(num, 1); + ASSERT_LE(num, kNumTables); + + vector<string> expected; + expected.insert(expected.end(), table_names.begin(), table_names.begin() + num); + + string filter = ""; + if (kNumTables != num) { + filter = Substitute("-tables=$0", JoinStrings(expected, ",")); + } + vector<string> lines; + NO_FATALS(RunActionStdoutLines( + Substitute("table list $0 $1", master_addr, filter), &lines)); + + std::sort(lines.begin(), lines.end()); + ASSERT_EQ(expected, lines); + }; + + const auto& ProcessTablets = [&] (const int num) { + ASSERT_GE(num, 1); + ASSERT_LE(num, kNumTables); + + string filter = ""; + if (kNumTables != num) { + filter = Substitute("-tables=$0", + JoinStringsIterator(table_names.begin(), table_names.begin() + num, ",")); + } + vector<string> lines; + NO_FATALS(RunActionStdoutLines( + Substitute("table list $0 $1 -list_tablets", master_addr, filter), &lines)); + + map<string, pair<string, string>> output; + for (int i = 0; i < lines.size(); ++i) { + if (lines[i].empty()) continue; + ASSERT_LE(i + 2, lines.size()); + output[lines[i]] = pair<string, string>(lines[i + 1], lines[i + 2]); + i += 2; + } + + for (const auto& e : output) { + shared_ptr<KuduTable> table; + ASSERT_OK(client->OpenTable(e.first, &table)); + vector<KuduScanToken*> tokens; + ElementDeleter deleter(&tokens); + KuduScanTokenBuilder builder(table.get()); + ASSERT_OK(builder.Build(&tokens)); + ASSERT_EQ(1, tokens.size()); // Only one partition(tablet) under table. + // Tablet's format. + string expect_tablet = Substitute(" T $0", tokens[0]->tablet().id()); + ASSERT_EQ(expect_tablet, e.second.first); + ASSERT_EQ(expect_replica, e.second.second); + } + }; + + // List the tables and tablets. + for (int i = 1; i <= kNumTables; ++i) { + ProcessTables(i); + ProcessTablets(i); + } +} + Status CreateLegacyHmsTable(HmsClient* client, const string& hms_database_name, const string& hms_table_name, http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_cluster.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc index 8fccd84..b6a6981 100644 --- a/src/kudu/tools/tool_action_cluster.cc +++ b/src/kudu/tools/tool_action_cluster.cc @@ -28,6 +28,7 @@ #include <boost/algorithm/string/predicate.hpp> #include <boost/optional/optional.hpp> #include <gflags/gflags.h> +#include <gflags/gflags_declare.h> #include <glog/logging.h> #include "kudu/gutil/basictypes.h" @@ -62,10 +63,7 @@ using strings::Substitute; } \ } while (0); -DEFINE_string(tables, "", - "Tables to check (comma-separated list of names). " - "If not specified, checks all tables."); - +DECLARE_string(tables); DEFINE_string(tablets, "", "Tablets to check (comma-separated list of IDs) " "If not specified, checks all tablets."); http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_common.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc index db86b02..e615915 100644 --- a/src/kudu/tools/tool_action_common.cc +++ b/src/kudu/tools/tool_action_common.cc @@ -56,6 +56,7 @@ #include "kudu/gutil/strings/split.h" #include "kudu/gutil/strings/stringpiece.h" #include "kudu/gutil/strings/substitute.h" +#include "kudu/gutil/strings/util.h" #include "kudu/master/master.proxy.h" // IWYU pragma: keep #include "kudu/rpc/messenger.h" #include "kudu/rpc/rpc_controller.h" @@ -101,6 +102,8 @@ DEFINE_string(flag_tags, "", "Comma-separated list of tags used to restrict whic "flags are returned. An empty value matches all tags"); DEFINE_bool(all_flags, false, "Whether to return all flags, or only flags that " "were explicitly set."); +DEFINE_string(tables, "", "Tables to include (comma-separated list of table names)" + "If not specified, includes all tables."); namespace boost { template <typename Signature> @@ -449,6 +452,16 @@ string GetMasterAddresses(const client::KuduClient& client) { return HostPort::ToCommaSeparatedString(client.data_->master_hostports()); } +bool MatchesAnyPattern(const vector<string>& patterns, const string& str) { + // Consider no filter a wildcard. + if (patterns.empty()) return true; + + for (const auto& p : patterns) { + if (MatchPattern(str, p)) return true; + } + return false; +} + Status PrintServerStatus(const string& address, uint16_t default_port) { ServerStatusPB status; RETURN_NOT_OK(GetServerStatus(address, default_port, &status)); http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_common.h ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h index 3043bd8..d3a4524 100644 --- a/src/kudu/tools/tool_action_common.h +++ b/src/kudu/tools/tool_action_common.h @@ -140,6 +140,10 @@ void SetAlterExternalCatalogs(client::KuduTableAlterer* alterer, bool alter_exte // Get the configured master addresses on the most recently connected to leader master. std::string GetMasterAddresses(const client::KuduClient& client); +// Return true if 'str' matches any of the patterns in 'patterns', or if +// 'patterns' is empty. +bool MatchesAnyPattern(const std::vector<std::string>& patterns, const std::string& str); + // A table of data to present to the user. // // Supports formatting based on the --format flag. http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_table.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc index 75d3a61..69521c2 100644 --- a/src/kudu/tools/tool_action_table.cc +++ b/src/kudu/tools/tool_action_table.cc @@ -23,6 +23,7 @@ #include <vector> #include <gflags/gflags.h> +#include <gflags/gflags_declare.h> #include "kudu/client/client.h" #include "kudu/client/replica_controller-internal.h" @@ -36,6 +37,7 @@ #include "kudu/tools/tool_action_common.h" #include "kudu/util/status.h" +DECLARE_string(tables); DEFINE_bool(alter_external_catalogs, true, "Whether to alter external catalogs, such as the Hive Metastore, " "when renaming a table."); @@ -73,7 +75,9 @@ class TableLister { vector<string> table_names; RETURN_NOT_OK(client->ListTables(&table_names)); + vector<string> table_filters = Split(FLAGS_tables, ",", strings::SkipEmpty()); for (const auto& tname : table_names) { + if (!MatchesAnyPattern(table_filters, tname)) continue; cout << tname << endl; if (!FLAGS_list_tablets) { continue; @@ -186,8 +190,9 @@ unique_ptr<Mode> BuildTableMode() { unique_ptr<Action> list_tables = ActionBuilder("list", &ListTables) - .Description("List all tables") + .Description("List tables") .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc }) + .AddOptionalParameter("tables") .AddOptionalParameter("list_tablets") .Build();