tool: add ksck This patch adds ksck functionality to the new CLI tool. ksck is still available as a library used by several integration tests.
Change-Id: Iad9aaffa5c0c77080dcaeb2cdfa572dcbeeb1407 Reviewed-on: http://gerrit.cloudera.org:8080/4121 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/86afadc7 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/86afadc7 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/86afadc7 Branch: refs/heads/master Commit: 86afadc7fc6f9274402edcb3599c243ec2de0ccb Parents: 7321b38 Author: Adar Dembo <[email protected]> Authored: Wed Aug 24 20:24:08 2016 -0700 Committer: Todd Lipcon <[email protected]> Committed: Thu Aug 25 21:46:47 2016 +0000 ---------------------------------------------------------------------- docs/release_notes.adoc | 3 + src/kudu/tools/CMakeLists.txt | 7 +- src/kudu/tools/kudu-ksck.cc | 151 ----------------------------- src/kudu/tools/kudu-tool-test.cc | 6 ++ src/kudu/tools/tool_action.h | 1 + src/kudu/tools/tool_action_cluster.cc | 149 ++++++++++++++++++++++++++++ src/kudu/tools/tool_main.cc | 1 + 7 files changed, 162 insertions(+), 156 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/docs/release_notes.adoc ---------------------------------------------------------------------- diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 38c3d7a..b629a2f 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -43,6 +43,9 @@ detailed below. - The `kudu-pbc-dump` tool has been removed. The same functionality is now implemented as `kudu pbc dump`. +- The `kudu-ksck` tool has been removed. The same functionality is now + implemented as `kudu cluster ksck`. + - KuduSession methods in the C++ library are no longer advertised as thread-safe to have one set of semantics for both C++ and Java Kudu client libraries. http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt index ce361f0..614bf11 100644 --- a/src/kudu/tools/CMakeLists.txt +++ b/src/kudu/tools/CMakeLists.txt @@ -82,13 +82,9 @@ target_link_libraries(ksck ${KUDU_BASE_LIBS} ) -add_executable(kudu-ksck kudu-ksck.cc) -target_link_libraries(kudu-ksck - ksck -) - add_executable(kudu tool_action.cc + tool_action_cluster.cc tool_action_fs.cc tool_action_pbc.cc tool_action_tablet.cc @@ -100,6 +96,7 @@ target_link_libraries(kudu kudu_common kudu_fs kudu_util + ksck master tserver ${KUDU_BASE_LIBS} http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-ksck.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-ksck.cc b/src/kudu/tools/kudu-ksck.cc deleted file mode 100644 index 70a6ce1..0000000 --- a/src/kudu/tools/kudu-ksck.cc +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Command line tool to run Ksck against a cluster. Defaults to running against a local Master -// on the default RPC port. It verifies that all the reported Tablet Servers are running and that -// the tablets are in a consistent state. - -#include <gflags/gflags.h> -#include <glog/logging.h> -#include <iostream> - -#include "kudu/gutil/strings/split.h" -#include "kudu/gutil/strings/substitute.h" -#include "kudu/tools/ksck_remote.h" -#include "kudu/util/flags.h" -#include "kudu/util/logging.h" - -#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \ - ::kudu::Status _s = (s); \ - if (PREDICT_FALSE(!_s.ok())) { \ - statuses->push_back(string(msg) + ": " + _s.ToString()); \ - } \ -} while (0); - -using std::cerr; -using std::cout; -using std::endl; -using std::shared_ptr; -using std::vector; -using strings::Substitute; - -DEFINE_string(master_address, "", - "Address of master server to run against."); - -DEFINE_bool(checksum_scan, false, - "Perform a checksum scan on data in the cluster."); - -DEFINE_string(tables, "", - "Tables to check (comma-separated list of names). " - "If not specified, checks all tables."); - -DEFINE_string(tablets, "", - "Tablets to check (comma-separated list of IDs) " - "If not specified, checks all tablets."); - -namespace kudu { -namespace tools { - -static string GetKsckUsage(const char* progname) { - string msg = Substitute("Usage: $0 --master_address=<addr> <flags>\n\n", progname); - msg += "Check the health of a Kudu cluster.\n\n" - "By default, ksck checks that master and tablet server processes are running,\n" - "and that table metadata is consistent. Use the 'checksum' flag to check that\n" - "tablet data is consistent (also see the 'tables' and 'tablets' flags below).\n" - "Use the 'checksum_snapshot' along with 'checksum' if the table or tablets are\n" - "actively receiving inserts or updates."; - return msg; -} - -// Run ksck. -// Error information is appended to the provided vector. -// If the vector is empty upon completion, ksck ran successfully. -static void RunKsck(vector<string>* error_messages) { - vector<Sockaddr> master_addrs; - PUSH_PREPEND_NOT_OK(ParseAddressList(FLAGS_master_address, - master::Master::kDefaultPort, - &master_addrs), - error_messages, "Unable to parse master address"); - // return early if the addresses could not be resolved - if (!error_messages->empty()) return; - - shared_ptr<KsckMaster> master; - PUSH_PREPEND_NOT_OK(RemoteKsckMaster::Build(master_addrs[0], &master), - error_messages, "Unable to build KsckMaster"); - if (!error_messages->empty()) return; - shared_ptr<KsckCluster> cluster(new KsckCluster(master)); - shared_ptr<Ksck> ksck(new Ksck(cluster)); - - ksck->set_table_filters(strings::Split(FLAGS_tables, ",", strings::SkipEmpty())); - ksck->set_tablet_id_filters(strings::Split(FLAGS_tablets, ",", strings::SkipEmpty())); - - // This is required for everything below. - PUSH_PREPEND_NOT_OK(ksck->CheckMasterRunning(), error_messages, - "Master aliveness check error"); - if (!error_messages->empty()) return; - - // This is also required for everything below. - PUSH_PREPEND_NOT_OK(ksck->FetchTableAndTabletInfo(), error_messages, - "Error fetching the cluster metadata from the Master server"); - if (!error_messages->empty()) return; - - PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages, - "Error fetching info from tablet servers"); - - // TODO: Add support for tables / tablets filter in the consistency check. - PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages, - "Table consistency check error"); - - if (FLAGS_checksum_scan) { - PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()), - error_messages, "Checksum scan error"); - } -} - -} // namespace tools -} // namespace kudu - -int main(int argc, char** argv) { - google::SetUsageMessage(kudu::tools::GetKsckUsage(argv[0])); - if (argc < 2) { - google::ShowUsageWithFlagsRestrict(argv[0], __FILE__); - exit(1); - } - kudu::ParseCommandLineFlags(&argc, &argv, true); - FLAGS_logtostderr = true; - kudu::InitGoogleLoggingSafe(argv[0]); - - vector<string> error_messages; - kudu::tools::RunKsck(&error_messages); - - // All good. - if (error_messages.empty()) { - cout << "OK" << endl; - return 0; - } - - // Something went wrong. - cerr << "==================" << endl; - cerr << "Errors:" << endl; - cerr << "==================" << endl; - for (const string& s : error_messages) { - cerr << s << endl; - } - cerr << endl; - cerr << "FAILED" << endl; - return 1; -} http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-tool-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 2700fb1..ba748aa 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -137,6 +137,12 @@ TEST_F(ToolTest, TestModeHelp) { }; NO_FATALS(RunTestHelp("tablet cmeta", kCmetaModeRegexes)); } + { + const vector<string> kClusterModeRegexes = { + "ksck.*Check the health of a Kudu cluster", + }; + NO_FATALS(RunTestHelp("cluster", kClusterModeRegexes)); + } } TEST_F(ToolTest, TestActionHelp) { http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action.h ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action.h b/src/kudu/tools/tool_action.h index da5d1e9..7d79a4f 100644 --- a/src/kudu/tools/tool_action.h +++ b/src/kudu/tools/tool_action.h @@ -247,6 +247,7 @@ class Action { }; // Returns new nodes for each major mode. +std::unique_ptr<Mode> BuildClusterMode(); std::unique_ptr<Mode> BuildFsMode(); std::unique_ptr<Mode> BuildPbcMode(); std::unique_ptr<Mode> BuildTabletMode(); http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action_cluster.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc new file mode 100644 index 0000000..d198a9b --- /dev/null +++ b/src/kudu/tools/tool_action_cluster.cc @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/tools/tool_action.h" + +#include <iostream> +#include <memory> +#include <string> +#include <utility> + +#include <gflags/gflags.h> + +#include "kudu/gutil/map-util.h" +#include "kudu/gutil/strings/split.h" +#include "kudu/master/master.h" +#include "kudu/tools/ksck.h" +#include "kudu/tools/ksck_remote.h" +#include "kudu/util/net/net_util.h" +#include "kudu/util/status.h" + +#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \ + ::kudu::Status _s = (s); \ + if (PREDICT_FALSE(!_s.ok())) { \ + statuses.push_back(string(msg) + ": " + _s.ToString()); \ + } \ +} while (0); + + +DEFINE_bool(checksum_scan, false, + "Perform a checksum scan on data in the cluster."); + +DEFINE_string(tables, "", + "Tables to check (comma-separated list of names). " + "If not specified, checks all tables."); + +DEFINE_string(tablets, "", + "Tablets to check (comma-separated list of IDs) " + "If not specified, checks all tablets."); + +using std::cerr; +using std::cout; +using std::endl; +using std::shared_ptr; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace kudu { +namespace tools { + +namespace { + +Status RunKsck(const RunnerContext& context) { + vector<Sockaddr> master_addrs; + string master_address = FindOrDie(context.required_args, "master_address"); + RETURN_NOT_OK_PREPEND(ParseAddressList(master_address, + master::Master::kDefaultPort, + &master_addrs), + "unable to parse master address"); + + shared_ptr<KsckMaster> master; + RETURN_NOT_OK_PREPEND(RemoteKsckMaster::Build(master_addrs[0], &master), + "unable to build KsckMaster"); + + shared_ptr<KsckCluster> cluster(new KsckCluster(master)); + shared_ptr<Ksck> ksck(new Ksck(cluster)); + + ksck->set_table_filters(strings::Split( + FLAGS_tables, ",", strings::SkipEmpty())); + ksck->set_tablet_id_filters(strings::Split( + FLAGS_tablets, ",", strings::SkipEmpty())); + + RETURN_NOT_OK_PREPEND(ksck->CheckMasterRunning(), + "master liveness check error"); + RETURN_NOT_OK_PREPEND(ksck->FetchTableAndTabletInfo(), + "error fetching the cluster metadata from the Master server"); + + vector<string> error_messages; + PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages, + "error fetching info from tablet servers"); + + // TODO: Add support for tables / tablets filter in the consistency check. + PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages, + "table consistency check error"); + + if (FLAGS_checksum_scan) { + PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()), + error_messages, "checksum scan error"); + } + + // All good. + if (error_messages.empty()) { + cout << "OK" << endl; + return Status::OK(); + } + + // Something went wrong. + cerr << "==================" << endl; + cerr << "Errors:" << endl; + cerr << "==================" << endl; + for (const auto& s : error_messages) { + cerr << s << endl; + } + cerr << endl; + cerr << "FAILED" << endl; + return Status::RuntimeError("ksck discovered errors"); +} + +} // anonymous namespace + +unique_ptr<Mode> BuildClusterMode() { + // TODO: the long description looks pretty horrible, need to fix. + string desc = "Check the health of a Kudu cluster. By default, ksck checks " + "that master and tablet server processes are running, and that table " + "metadata is consistent. Use the 'checksum' flag to check that tablet " + "data is consistent (also see the 'tables' and 'tablets' flags). Use the " + "'checksum_snapshot' along with 'checksum' if the table or tablets are " + "actively receiving inserts or updates."; + unique_ptr<Action> ksck = ActionBuilder( + { "ksck", desc }, &RunKsck) + .AddRequiredParameter({ "master_address", "Kudu Master RPC address of form hostname:port" }) + .AddOptionalParameter("checksum_scan") + .AddOptionalParameter("checksum_snapshot") + .AddOptionalParameter("tables") + .AddOptionalParameter("tablets") + .Build(); + + return ModeBuilder({ "cluster", "Operate on a Kudu cluster" }) + .AddAction(std::move(ksck)) + .Build(); +} + +} // namespace tools +} // namespace kudu + http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_main.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_main.cc b/src/kudu/tools/tool_main.cc index 18137a1..70fdfc0 100644 --- a/src/kudu/tools/tool_main.cc +++ b/src/kudu/tools/tool_main.cc @@ -111,6 +111,7 @@ int DispatchCommand(const vector<Mode*>& chain, int RunTool(int argc, char** argv, bool show_help) { unique_ptr<Mode> root = ModeBuilder({ argv[0], "" }) // root mode description isn't printed + .AddMode(BuildClusterMode()) .AddMode(BuildFsMode()) .AddMode(BuildPbcMode()) .AddMode(BuildTabletMode())
