Repository: kudu Updated Branches: refs/heads/master 7fee0ec15 -> 94b066bb3
[tools] Add tool to dump memtrackers This adds a new tool, `kudu diagnose dump_mem_trackers`, that dumps the mem-trackers information. It contains information equivalent to the tracker table on /mem-trackers. There's a new RPC introduced to support this tool. Output both as JSON and as a table is supported. The table output type flattens the tracker hierarchy, but it can be reconstructed from the 'id' and 'parent_id' columns. Some Kudu community members have reported to me they have started using the /mem-trackers page to investigate performance and memory issues with Kudu clusters, and having a way to get the same information in a form amenable to programmatic searching, sorting, and filtering is convenient. This tool can also serve as a starting point for more automated analysis of the tracker information. The particular advantage of the memtracker dump over, say, the data generated by heap sampling is that the memtracker shows where memory is owned. The sampling just shows where it was allocated in a call stack. So, for example, using the memtracker information it's possible to tell very quickly exactly which tablets are using a lot of memory in DeltaMemStores. Heap samples would indicate memory allocated in update-related code paths, but it would not be possible to tell which tablet replicas actually held the memory that was allocated. Here's a small sample of output in csv table format: tablet-2e02d00d46834b359c6ba0f8d471cbf9,server,-1,2318,2583 txn_tracker,tablet-2e02d00d46834b359c6ba0f8d471cbf9,67108864,0,0 MemRowSet-1,tablet-2e02d00d46834b359c6ba0f8d471cbf9,-1,265,265 DeltaMemStores,tablet-2e02d00d46834b359c6ba0f8d471cbf9,-1,265,265 tablet-2d40450f6556485eab92f64f90756b61,server,-1,2318,2583 txn_tracker,tablet-2d40450f6556485eab92f64f90756b61,67108864,0,0 MemRowSet-1,tablet-2d40450f6556485eab92f64f90756b61,-1,265,265 DeltaMemStores,tablet-2d40450f6556485eab92f64f90756b61,-1,265,265 The columns are id,parent-id,limit,current_consumption,peak_consumption just like the table on /mem-trackers. Change-Id: I3e54f809bb6434b8d8e8c95771fe089c9da122d0 Reviewed-on: http://gerrit.cloudera.org:8080/11975 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e60e9bfe Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e60e9bfe Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e60e9bfe Branch: refs/heads/master Commit: e60e9bfe6e3a81f90639f74c4a10790e46938d03 Parents: 7fee0ec Author: Will Berkeley <[email protected]> Authored: Wed Nov 21 04:08:26 2018 -0800 Committer: Will Berkeley <[email protected]> Committed: Thu Nov 29 22:21:24 2018 +0000 ---------------------------------------------------------------------- src/kudu/server/generic_service.cc | 8 ++ src/kudu/server/generic_service.h | 21 ++-- src/kudu/server/server_base.proto | 10 ++ src/kudu/tools/kudu-admin-test.cc | 53 +++++++++++ src/kudu/tools/kudu-tool-test.cc | 2 + src/kudu/tools/tool_action_common.cc | 48 ++++++++++ src/kudu/tools/tool_action_common.h | 5 + src/kudu/tools/tool_action_diagnose.cc | 7 +- src/kudu/tools/tool_action_master.cc | 15 +++ src/kudu/tools/tool_action_tserver.cc | 15 +++ src/kudu/util/CMakeLists.txt | 15 +++ src/kudu/util/mem_tracker-test.cc | 143 ++++++++++++++++++++++++++++ src/kudu/util/mem_tracker.cc | 35 ++++++- src/kudu/util/mem_tracker.h | 7 +- src/kudu/util/mem_tracker.proto | 39 ++++++++ 15 files changed, 411 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/server/generic_service.cc ---------------------------------------------------------------------- diff --git a/src/kudu/server/generic_service.cc b/src/kudu/server/generic_service.cc index 0b8fc59..10e6ed4 100644 --- a/src/kudu/server/generic_service.cc +++ b/src/kudu/server/generic_service.cc @@ -42,6 +42,7 @@ #include "kudu/util/debug/leak_annotations.h" // IWYU pragma: keep #include "kudu/util/flag_tags.h" #include "kudu/util/flags.h" +#include "kudu/util/mem_tracker.h" #include "kudu/util/status.h" DECLARE_string(time_source); @@ -251,5 +252,12 @@ void GenericServiceImpl::GetStatus(const GetStatusRequestPB* /*req*/, rpc->RespondSuccess(); } +void GenericServiceImpl::DumpMemTrackers(const DumpMemTrackersRequestPB* /*req*/, + DumpMemTrackersResponsePB* resp, + rpc::RpcContext* rpc) { + MemTracker::TrackersToPb(resp->mutable_root_tracker()); + rpc->RespondSuccess(); +} + } // namespace server } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/server/generic_service.h ---------------------------------------------------------------------- diff --git a/src/kudu/server/generic_service.h b/src/kudu/server/generic_service.h index 4cf0f79..283968f 100644 --- a/src/kudu/server/generic_service.h +++ b/src/kudu/server/generic_service.h @@ -18,7 +18,6 @@ #define KUDU_SERVER_GENERIC_SERVICE_H #include "kudu/gutil/macros.h" -#include "kudu/gutil/port.h" #include "kudu/server/server_base.service.h" namespace google { @@ -37,6 +36,8 @@ namespace server { class CheckLeaksRequestPB; class CheckLeaksResponsePB; +class DumpMemTrackersRequestPB; +class DumpMemTrackersResponsePB; class FlushCoverageRequestPB; class FlushCoverageResponsePB; class GetFlagsRequestPB; @@ -66,31 +67,35 @@ class GenericServiceImpl : public GenericServiceIf { virtual void GetFlags(const GetFlagsRequestPB* req, GetFlagsResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; virtual void SetFlag(const SetFlagRequestPB* req, SetFlagResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; virtual void FlushCoverage(const FlushCoverageRequestPB* req, FlushCoverageResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; virtual void CheckLeaks(const CheckLeaksRequestPB* req, CheckLeaksResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; virtual void ServerClock(const ServerClockRequestPB* req, ServerClockResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; virtual void SetServerWallClockForTests(const SetServerWallClockForTestsRequestPB *req, SetServerWallClockForTestsResponsePB *resp, - rpc::RpcContext *context) OVERRIDE; + rpc::RpcContext *context) override; virtual void GetStatus(const GetStatusRequestPB* req, GetStatusResponsePB* resp, - rpc::RpcContext* rpc) OVERRIDE; + rpc::RpcContext* rpc) override; + + virtual void DumpMemTrackers(const DumpMemTrackersRequestPB* req, + DumpMemTrackersResponsePB* resp, + rpc::RpcContext* rpc) override; private: ServerBase* server_; http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/server/server_base.proto ---------------------------------------------------------------------- diff --git a/src/kudu/server/server_base.proto b/src/kudu/server/server_base.proto index cf680dd..478e758 100644 --- a/src/kudu/server/server_base.proto +++ b/src/kudu/server/server_base.proto @@ -22,6 +22,7 @@ option java_package = "org.apache.kudu.server"; import "kudu/common/common.proto"; import "kudu/common/wire_protocol.proto"; import "kudu/rpc/rpc_header.proto"; +import "kudu/util/mem_tracker.proto"; import "kudu/util/version_info.proto"; // The status information dumped by a server after it starts. @@ -145,6 +146,12 @@ message SetServerWallClockForTestsResponsePB { required bool success = 1; } +message DumpMemTrackersRequestPB { +} +message DumpMemTrackersResponsePB { + optional MemTrackerPB root_tracker = 1; +} + service GenericService { option (kudu.rpc.default_authz_method) = "AuthorizeSuperUser"; @@ -171,4 +178,7 @@ service GenericService { rpc GetStatus(GetStatusRequestPB) returns (GetStatusResponsePB) { option (kudu.rpc.authz_method) = "AuthorizeClient"; } + + rpc DumpMemTrackers(DumpMemTrackersRequestPB) + returns (DumpMemTrackersResponsePB); } http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/kudu-admin-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc index fb4bfbb..a870026 100644 --- a/src/kudu/tools/kudu-admin-test.cc +++ b/src/kudu/tools/kudu-admin-test.cc @@ -57,11 +57,13 @@ #include "kudu/integration-tests/test_workload.h" #include "kudu/integration-tests/ts_itest-base.h" #include "kudu/master/master.pb.h" +#include "kudu/master/sys_catalog.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/tablet/metadata.pb.h" #include "kudu/tools/tool_test_util.h" #include "kudu/tserver/tablet_server-test-base.h" #include "kudu/util/monotime.h" +#include "kudu/util/net/net_util.h" #include "kudu/util/net/sockaddr.h" #include "kudu/util/pb_util.h" #include "kudu/util/status.h" @@ -2055,5 +2057,56 @@ TEST_F(AdminCliTest, TestLocateRowAndCheckRowPresence) { ASSERT_STR_CONTAINS(stdout, row_str); }); } + +TEST_F(AdminCliTest, TestDumpMemTrackers) { + FLAGS_num_tablet_servers = 1; + FLAGS_num_replicas = 1; + + NO_FATALS(BuildAndStart()); + + // Grab list of tablet_ids from any tserver so we can check the output. + vector<TServerDetails*> tservers; + vector<string> tablet_ids; + AppendValuesFromMap(tablet_servers_, &tservers); + ListRunningTabletIds(tservers.front(), + MonoDelta::FromSeconds(30), + &tablet_ids); + ASSERT_EQ(1, tablet_ids.size()); + const string& tablet_id = tablet_ids[0]; + + // The tool should work against the master. + string stdout, stderr; + Status s = RunKuduTool({ + "master", + "dump_memtrackers", + cluster_->master()->bound_rpc_hostport().ToString(), + "-format=csv", + }, &stdout, &stderr); + ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr); + + // The memtrackers dump from the master should contain a tracker for the + // systablet that has 'server' as its parent. + ASSERT_STR_CONTAINS( + stdout, + Substitute("tablet-$0,server", + master::SysCatalogTable::kSysCatalogTabletId)); + + // The tool should work against the tablet server. + stdout.clear(); + stderr.clear(); + s = RunKuduTool({ + "tserver", + "dump_memtrackers", + cluster_->tablet_server(0)->bound_rpc_hostport().ToString(), + "-memtracker_output=json_compact", + }, &stdout, &stderr); + ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr); + + // The memtrackers dump from the tablet server should contain a tracker for + // the tablet and some tracker that is a child of that tracker. + const string tablet_tracker_id = Substitute("tablet-$0", tablet_id); + ASSERT_STR_CONTAINS(stdout, Substitute("\"id\":\"$0\"", tablet_tracker_id)); + ASSERT_STR_CONTAINS(stdout, Substitute("\"parent_id\":\"$0\"", tablet_tracker_id)); +} } // namespace tools } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/kudu-tool-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 659c1ee..0a158c7 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -552,6 +552,7 @@ TEST_F(ToolTest, TestModeHelp) { } { const vector<string> kMasterModeRegexes = { + "dump_memtrackers.*Dump the memtrackers", "get_flags.*Get the gflags", "set_flag.*Change a gflag value", "status.*Get the status", @@ -615,6 +616,7 @@ TEST_F(ToolTest, TestModeHelp) { } { const vector<string> kTServerModeRegexes = { + "dump_memtrackers.*Dump the memtrackers", "get_flags.*Get the gflags", "set_flag.*Change a gflag value", "status.*Get the status", http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/tool_action_common.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc index 22e5b66..5a32a77 100644 --- a/src/kudu/tools/tool_action_common.cc +++ b/src/kudu/tools/tool_action_common.cc @@ -27,6 +27,7 @@ #include <iterator> #include <memory> #include <numeric> +#include <stack> #include <string> #include <unordered_map> #include <vector> @@ -70,6 +71,7 @@ #include "kudu/tserver/tserver_service.proxy.h" // IWYU pragma: keep #include "kudu/util/faststring.h" #include "kudu/util/jsonwriter.h" +#include "kudu/util/mem_tracker.pb.h" #include "kudu/util/memory/arena.h" #include "kudu/util/monotime.h" #include "kudu/util/net/net_util.h" @@ -107,6 +109,10 @@ DEFINE_bool(all_flags, false, "Whether to return all flags, or only flags that " DEFINE_string(tables, "", "Tables to include (comma-separated list of table names)" "If not specified, includes all tables."); +DEFINE_string(memtracker_output, "table", + "One of 'json', 'json_compact' or 'table'. Table output flattens " + "the memtracker hierarchy."); + namespace boost { template <typename Signature> class function; @@ -479,6 +485,48 @@ Status PrintServerTimestamp(const string& address, uint16_t default_port) { return Status::OK(); } +Status DumpMemTrackers(const string& address, uint16_t default_port) { + unique_ptr<GenericServiceProxy> proxy; + RETURN_NOT_OK(BuildProxy(address, default_port, &proxy)); + + server::DumpMemTrackersRequestPB req; + server::DumpMemTrackersResponsePB resp; + RpcController rpc; + rpc.set_timeout(MonoDelta::FromMilliseconds(FLAGS_timeout_ms)); + RETURN_NOT_OK(proxy->DumpMemTrackers(req, &resp, &rpc)); + + if (boost::iequals(FLAGS_memtracker_output, "json")) { + cout << JsonWriter::ToJson(resp.root_tracker(), JsonWriter::Mode::PRETTY) + << endl; + } else if (boost::iequals(FLAGS_memtracker_output, "json_compact")) { + cout << JsonWriter::ToJson(resp.root_tracker(), JsonWriter::Mode::COMPACT) + << endl; + } else if (boost::iequals(FLAGS_memtracker_output, "table")) { + DataTable table({ "id", "parent_id", "limit", + "current consumption", "peak_consumption" }); + const auto& root = resp.root_tracker(); + std::stack<const MemTrackerPB*> to_process; + to_process.push(&root); + while (!to_process.empty()) { + const auto* tracker = to_process.top(); + to_process.pop(); + table.AddRow({ tracker->id(), + tracker->has_parent_id() ? tracker->parent_id() : "<none>", + std::to_string(tracker->limit()), + std::to_string(tracker->current_consumption()), + std::to_string(tracker->peak_consumption()) }); + for (const auto& child_tracker : tracker->child_trackers()) { + to_process.push(&child_tracker); + } + } + RETURN_NOT_OK(table.PrintTo(cout)); + } else { + return Status::InvalidArgument("unknown output type (--memtracker_output)", + FLAGS_memtracker_output); + } + return Status::OK(); +} + namespace { // Pretty print a table using the psql format. For example: http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/tool_action_common.h ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h index b90e12b..6040049 100644 --- a/src/kudu/tools/tool_action_common.h +++ b/src/kudu/tools/tool_action_common.h @@ -133,6 +133,11 @@ Status PrintServerFlags(const std::string& address, uint16_t default_port); Status SetServerFlag(const std::string& address, uint16_t default_port, const std::string& flag, const std::string& value); +// Dump the memtrackers of the server at 'address'. +// +// If 'address' does not contain a port, 'default_port' will be used instead. +Status DumpMemTrackers(const std::string& address, uint16_t default_port); + // Return true if 'str' matches any of the patterns in 'patterns', or if // 'patterns' is empty. bool MatchesAnyPattern(const std::vector<std::string>& patterns, const std::string& str); http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/tool_action_diagnose.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_diagnose.cc b/src/kudu/tools/tool_action_diagnose.cc index 3b84441..932c608 100644 --- a/src/kudu/tools/tool_action_diagnose.cc +++ b/src/kudu/tools/tool_action_diagnose.cc @@ -16,7 +16,6 @@ // under the License. #include <algorithm> -#include <array> #include <cerrno> #include <fstream> // IWYU pragma: keep #include <memory> @@ -24,16 +23,20 @@ #include <unordered_map> #include <vector> +#include <gflags/gflags_declare.h> + #include "kudu/gutil/strings/substitute.h" #include "kudu/tools/diagnostics_log_parser.h" #include "kudu/tools/tool_action.h" #include "kudu/util/errno.h" #include "kudu/util/status.h" +DECLARE_int64(timeout_ms); +DECLARE_string(format); + namespace kudu { namespace tools { -using std::array; using std::ifstream; using std::string; using std::unique_ptr; http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/tool_action_master.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_master.cc b/src/kudu/tools/tool_action_master.cc index ae07172..ea263fe 100644 --- a/src/kudu/tools/tool_action_master.cc +++ b/src/kudu/tools/tool_action_master.cc @@ -154,9 +154,23 @@ Status ListMasters(const RunnerContext& context) { return Status::OK(); } +Status MasterDumpMemTrackers(const RunnerContext& context) { + const auto& address = FindOrDie(context.required_args, kMasterAddressArg); + return DumpMemTrackers(address, master::Master::kDefaultPort); +} + } // anonymous namespace unique_ptr<Mode> BuildMasterMode() { + unique_ptr<Action> dump_memtrackers = + ActionBuilder("dump_memtrackers", &MasterDumpMemTrackers) + .Description("Dump the memtrackers from a Kudu Master") + .AddRequiredParameter({ kMasterAddressArg, kMasterAddressDesc }) + .AddOptionalParameter("format") + .AddOptionalParameter("memtracker_output") + .AddOptionalParameter("timeout_ms") + .Build(); + unique_ptr<Action> get_flags = ActionBuilder("get_flags", &MasterGetFlags) .Description("Get the gflags for a Kudu Master") @@ -200,6 +214,7 @@ unique_ptr<Mode> BuildMasterMode() { return ModeBuilder("master") .Description("Operate on a Kudu Master") + .AddAction(std::move(dump_memtrackers)) .AddAction(std::move(get_flags)) .AddAction(std::move(set_flag)) .AddAction(std::move(status)) http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/tools/tool_action_tserver.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/tool_action_tserver.cc b/src/kudu/tools/tool_action_tserver.cc index 7e2f261..58b341e 100644 --- a/src/kudu/tools/tool_action_tserver.cc +++ b/src/kudu/tools/tool_action_tserver.cc @@ -153,9 +153,23 @@ Status ListTServers(const RunnerContext& context) { return Status::OK(); } +Status TserverDumpMemTrackers(const RunnerContext& context) { + const auto& address = FindOrDie(context.required_args, kTServerAddressArg); + return DumpMemTrackers(address, tserver::TabletServer::kDefaultPort); +} + } // anonymous namespace unique_ptr<Mode> BuildTServerMode() { + unique_ptr<Action> dump_memtrackers = + ActionBuilder("dump_memtrackers", &TserverDumpMemTrackers) + .Description("Dump the memtrackers from a Kudu Tablet Server") + .AddRequiredParameter({ kTServerAddressArg, kTServerAddressDesc }) + .AddOptionalParameter("format") + .AddOptionalParameter("memtracker_output") + .AddOptionalParameter("timeout_ms") + .Build(); + unique_ptr<Action> get_flags = ActionBuilder("get_flags", &TServerGetFlags) .Description("Get the gflags for a Kudu Tablet Server") @@ -200,6 +214,7 @@ unique_ptr<Mode> BuildTServerMode() { return ModeBuilder("tserver") .Description("Operate on a Kudu Tablet Server") + .AddAction(std::move(dump_memtrackers)) .AddAction(std::move(get_flags)) .AddAction(std::move(set_flag)) .AddAction(std::move(status)) http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/util/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt index bf66dd8..8779ac1 100644 --- a/src/kudu/util/CMakeLists.txt +++ b/src/kudu/util/CMakeLists.txt @@ -72,6 +72,20 @@ ADD_EXPORTABLE_LIBRARY(maintenance_manager_proto NONLINK_DEPS ${MAINTENANCE_MANAGER_PROTO_TGTS}) ####################################### +# mem_tracker_proto +####################################### + +PROTOBUF_GENERATE_CPP( + MEM_TRACKER_PROTO_SRCS MEM_TRACKER_PROTO_HDRS MEM_TRACKER_PROTO_TGTS + SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../.. + BINARY_ROOT ${CMAKE_CURRENT_BINARY_DIR}/../.. + PROTO_FILES mem_tracker.proto) +ADD_EXPORTABLE_LIBRARY(mem_tracker_proto + SRCS ${MEM_TRACKER_PROTO_SRCS} + DEPS protobuf + NONLINK_DEPS ${MEM_TRACKER_PROTO_TGTS}) + +####################################### # pb_util_proto ####################################### @@ -238,6 +252,7 @@ set(UTIL_LIBS histogram_proto libev maintenance_manager_proto + mem_tracker_proto pb_util_proto protobuf version_info_proto http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/util/mem_tracker-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/mem_tracker-test.cc b/src/kudu/util/mem_tracker-test.cc index dbadd09..900f2f9 100644 --- a/src/kudu/util/mem_tracker-test.cc +++ b/src/kudu/util/mem_tracker-test.cc @@ -31,7 +31,10 @@ #include <gtest/gtest.h> #include "kudu/gutil/strings/substitute.h" +#include "kudu/util/jsonwriter.h" +#include "kudu/util/mem_tracker.pb.h" #include "kudu/util/monotime.h" +#include "kudu/util/scoped_cleanup.h" #include "kudu/util/test_util.h" namespace kudu { @@ -117,6 +120,146 @@ TEST(MemTrackerTest, TrackerHierarchy) { c2->Release(60); } +TEST(MemTrackerTest, TestToPb) { + shared_ptr<MemTracker> p0 = MemTracker::CreateTracker(100, "p0"); + shared_ptr<MemTracker> c00 = MemTracker::CreateTracker(80, "c00", p0); + shared_ptr<MemTracker> gc000 = MemTracker::CreateTracker(40, "c000", c00); + shared_ptr<MemTracker> c01 = MemTracker::CreateTracker(40, "c01", p0); + shared_ptr<MemTracker> p1 = MemTracker::CreateTracker(120, "p1"); + shared_ptr<MemTracker> c10 = MemTracker::CreateTracker(70, "c10", p1); + + MemTrackerPB pb; + MemTracker::TrackersToPb(&pb); + const auto expected = R"({ + "id": "root", + "limit": -1, + "current_consumption": 0, + "peak_consumption": 110, + "child_trackers": [ + { + "id": "p0", + "parent_id": "root", + "limit": 100, + "current_consumption": 0, + "peak_consumption": 0, + "child_trackers": [ + { + "id": "c00", + "parent_id": "p0", + "limit": 80, + "current_consumption": 0, + "peak_consumption": 0, + "child_trackers": [ + { + "id": "c000", + "parent_id": "c00", + "limit": 40, + "current_consumption": 0, + "peak_consumption": 0 + } + ] + }, + { + "id": "c01", + "parent_id": "p0", + "limit": 40, + "current_consumption": 0, + "peak_consumption": 0 + } + ] + }, + { + "id": "p1", + "parent_id": "root", + "limit": 120, + "current_consumption": 0, + "peak_consumption": 0, + "child_trackers": [ + { + "id": "c10", + "parent_id": "p1", + "limit": 70, + "current_consumption": 0, + "peak_consumption": 0 + } + ] + } + ] +})"; + ASSERT_EQ(expected, JsonWriter::ToJson(pb, JsonWriter::Mode::PRETTY)); + + // Tickle the 'consumption' and 'peak_consumption' values a bit. + gc000->Consume(40); + gc000->Release(20); + + // Need to release all memory before the root MemTracker is destroyed, even + // if the test fails. + SCOPED_CLEANUP({ + gc000->Release(20); + }); + + pb.Clear(); + MemTracker::TrackersToPb(&pb); + const auto expected_post_tickling = R"({ + "id": "root", + "limit": -1, + "current_consumption": 20, + "peak_consumption": 110, + "child_trackers": [ + { + "id": "p0", + "parent_id": "root", + "limit": 100, + "current_consumption": 20, + "peak_consumption": 40, + "child_trackers": [ + { + "id": "c00", + "parent_id": "p0", + "limit": 80, + "current_consumption": 20, + "peak_consumption": 40, + "child_trackers": [ + { + "id": "c000", + "parent_id": "c00", + "limit": 40, + "current_consumption": 20, + "peak_consumption": 40 + } + ] + }, + { + "id": "c01", + "parent_id": "p0", + "limit": 40, + "current_consumption": 0, + "peak_consumption": 0 + } + ] + }, + { + "id": "p1", + "parent_id": "root", + "limit": 120, + "current_consumption": 0, + "peak_consumption": 0, + "child_trackers": [ + { + "id": "c10", + "parent_id": "p1", + "limit": 70, + "current_consumption": 0, + "peak_consumption": 0 + } + ] + } + ] +})"; + ASSERT_EQ(expected_post_tickling, + JsonWriter::ToJson(pb, JsonWriter::Mode::PRETTY)); +} + class GcFunctionHelper { public: static const int kNumReleaseBytes = 1; http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/util/mem_tracker.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/mem_tracker.cc b/src/kudu/util/mem_tracker.cc index f7294d1..8e503a7 100644 --- a/src/kudu/util/mem_tracker.cc +++ b/src/kudu/util/mem_tracker.cc @@ -24,10 +24,13 @@ #include <list> #include <memory> #include <ostream> +#include <stack> +#include <type_traits> #include "kudu/gutil/once.h" #include "kudu/gutil/port.h" #include "kudu/gutil/strings/substitute.h" +#include "kudu/util/mem_tracker.pb.h" #include "kudu/util/mutex.h" #include "kudu/util/process_memory.h" @@ -37,6 +40,7 @@ namespace kudu { // somewhat from kudu. using std::deque; +using std::stack; using std::list; using std::shared_ptr; using std::string; @@ -169,7 +173,7 @@ shared_ptr<MemTracker> MemTracker::FindOrCreateGlobalTracker( void MemTracker::ListTrackers(vector<shared_ptr<MemTracker>>* trackers) { trackers->clear(); - deque<shared_ptr<MemTracker> > to_process; + deque<shared_ptr<MemTracker>> to_process; to_process.push_front(GetRootTracker()); while (!to_process.empty()) { shared_ptr<MemTracker> t = to_process.back(); @@ -188,6 +192,35 @@ void MemTracker::ListTrackers(vector<shared_ptr<MemTracker>>* trackers) { } } +void MemTracker::TrackersToPb(MemTrackerPB* pb) { + CHECK(pb); + stack<std::pair<shared_ptr<MemTracker>, MemTrackerPB*>> to_process; + to_process.emplace(std::make_pair(GetRootTracker(), pb)); + while (!to_process.empty()) { + auto tracker_and_pb = std::move(to_process.top()); + to_process.pop(); + auto& tracker = tracker_and_pb.first; + auto* tracker_pb = tracker_and_pb.second; + tracker_pb->set_id(tracker->id()); + if (tracker->parent()) { + tracker_pb->set_parent_id(tracker->parent()->id()); + } + tracker_pb->set_limit(tracker->limit()); + tracker_pb->set_current_consumption(tracker->consumption()); + tracker_pb->set_peak_consumption(tracker->peak_consumption()); + { + MutexLock l(tracker->child_trackers_lock_); + for (const auto& child_weak : tracker->child_trackers_) { + shared_ptr<MemTracker> child = child_weak.lock(); + if (child) { + auto* child_pb = tracker_pb->add_child_trackers(); + to_process.emplace(std::make_pair(std::move(child), child_pb)); + } + } + } + } +} + void MemTracker::Consume(int64_t bytes) { if (bytes < 0) { Release(-bytes); http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/util/mem_tracker.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/mem_tracker.h b/src/kudu/util/mem_tracker.h index 14db374..b4192d7 100644 --- a/src/kudu/util/mem_tracker.h +++ b/src/kudu/util/mem_tracker.h @@ -31,6 +31,8 @@ namespace kudu { +class MemTrackerPB; + // A MemTracker tracks memory consumption; it contains an optional limit and is // arranged into a tree structure such that the consumption tracked by a // MemTracker is also tracked by its ancestors. @@ -94,7 +96,10 @@ class MemTracker : public std::enable_shared_from_this<MemTracker> { int64_t byte_limit, const std::string& id); // Returns a list of all the valid trackers. - static void ListTrackers(std::vector<std::shared_ptr<MemTracker> >* trackers); + static void ListTrackers(std::vector<std::shared_ptr<MemTracker>>* trackers); + + // Marshals the tracker tree into 'pb'. + static void TrackersToPb(MemTrackerPB* pb); // Gets a shared_ptr to the "root" tracker, creating it if necessary. static std::shared_ptr<MemTracker> GetRootTracker(); http://git-wip-us.apache.org/repos/asf/kudu/blob/e60e9bfe/src/kudu/util/mem_tracker.proto ---------------------------------------------------------------------- diff --git a/src/kudu/util/mem_tracker.proto b/src/kudu/util/mem_tracker.proto new file mode 100644 index 0000000..75d3824 --- /dev/null +++ b/src/kudu/util/mem_tracker.proto @@ -0,0 +1,39 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +syntax = "proto2"; +package kudu; + +option java_package = "org.apache.kudu"; + +// A snapshot of a Kudu server mem_tracker tree. +// Note that recursive messages are not a good idea in general because +// serializing a message requires knowing the size of its sub-messages. +// However, the mem_tracker tree is a wide and shallow tree, with a maximum +// depth of 3 realized by, e.g., the path +// +// root -> server -> tablet-<id> -> DeltaMemStores +// +// This could change in the future, however. +message MemTrackerPB { + optional string id = 1; + optional string parent_id = 2; + optional int64 limit = 3; + optional int64 current_consumption = 4; + optional int64 peak_consumption = 5; + repeated MemTrackerPB child_trackers = 6; +}
