This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch migrate-metrics-dev
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/migrate-metrics-dev by this
push:
new e0d192036 feat(new_metrics): migrate metrics for replica_stub (part 2)
(#1459)
e0d192036 is described below
commit e0d19203640935ce950b2c8ec5e19dd75bc56c49
Author: Dan Wang <[email protected]>
AuthorDate: Wed Apr 26 11:03:06 2023 +0800
feat(new_metrics): migrate metrics for replica_stub (part 2) (#1459)
https://github.com/apache/incubator-pegasus/issues/1454
This is the 2nd part of migrating metrics of `replica_stub` to new
framework,
all of which are learn-related.
During this migration, there are 3 metrics still keeping server-level,
including
the number of learning replicas, the max duration and the max size of files
that
are copied from learnee among all learning replicas.
Another 11 metrics are changed from server-level to replica-level, since
they
should be observed for each replica. All of them are observed from the view
of learners, namely the potential secondary replica. The learnee is the
primary
replica. These metrics include: the number of learns launched by learner,
the
number of learn rounds launched by learner (during a learn there might be
multiple rounds, the number of files that are copied from learnee, the size
of
files that are copied from learnee, the size of data that are copied from
learnee's
buffer, the number of learn responses of `LT_CACHE`, `LT_APP` and `LT_LOG`
type decided by learner with each learn response related to an `RPC_LEARN`
request, the number of times learner resets its local state (since its
local state
is newer than learnee's) with each reset related to an learn response of an
`RPC_LEARN` request, the number of failed and successful learns launched by
learner.
---
src/replica/replica.cpp | 75 ++++++++++++++++++++++++++++++++-
src/replica/replica.h | 12 ++++++
src/replica/replica_learn.cpp | 39 ++++++++++-------
src/replica/replica_stub.cpp | 98 ++++++++++---------------------------------
src/replica/replica_stub.h | 17 ++------
src/utils/metrics.h | 4 ++
6 files changed, 141 insertions(+), 104 deletions(-)
diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp
index aceb4770a..7348a5ba7 100644
--- a/src/replica/replica.cpp
+++ b/src/replica/replica.cpp
@@ -126,6 +126,68 @@ METRIC_DEFINE_counter(replica,
dsn::metric_unit::kRequests,
"The number of rejected non-idempotent write requests by
duplication");
+METRIC_DEFINE_counter(
+ replica,
+ learn_count,
+ dsn::metric_unit::kLearns,
+ "The number of learns launched by learner (i.e. potential secondary
replica)");
+
+METRIC_DEFINE_counter(replica,
+ learn_rounds,
+ dsn::metric_unit::kRounds,
+ "The number of learn rounds launched by learner (during
a learn there might"
+ "be multiple rounds)");
+
+METRIC_DEFINE_counter(replica,
+ learn_copy_files,
+ dsn::metric_unit::kFiles,
+ "The number of files that are copied from learnee (i.e.
primary replica)");
+
+METRIC_DEFINE_counter(replica,
+ learn_copy_file_bytes,
+ dsn::metric_unit::kBytes,
+ "The size of file that are copied from learnee");
+
+METRIC_DEFINE_counter(replica,
+ learn_copy_buffer_bytes,
+ dsn::metric_unit::kBytes,
+ "The size of data that are copied from learnee's
buffer");
+
+METRIC_DEFINE_counter(replica,
+ learn_lt_cache_responses,
+ dsn::metric_unit::kResponses,
+ "The number of learn responses of LT_CACHE type decided
by learner, with "
+ "each learn response related to an RPC_LEARN request");
+
+METRIC_DEFINE_counter(replica,
+ learn_lt_app_responses,
+ dsn::metric_unit::kResponses,
+ "The number of learn responses of LT_APP type decided by
learner, with each "
+ "learn response related to an RPC_LEARN request");
+
+METRIC_DEFINE_counter(replica,
+ learn_lt_log_responses,
+ dsn::metric_unit::kResponses,
+ "The number of learn responses of LT_LOG type decided by
learner, with each "
+ "learn response related to an RPC_LEARN request");
+
+METRIC_DEFINE_counter(replica,
+ learn_resets,
+ dsn::metric_unit::kResets,
+ "The number of times learner resets its local state
(since its local state "
+ "is newer than learnee's), with each reset related to an
learn response of "
+ "an RPC_LEARN request");
+
+METRIC_DEFINE_counter(replica,
+ learn_failed_count,
+ dsn::metric_unit::kLearns,
+ "The number of failed learns launched by learner");
+
+METRIC_DEFINE_counter(replica,
+ learn_successful_count,
+ dsn::metric_unit::kLearns,
+ "The number of successful learns launched by learner");
+
namespace dsn {
namespace replication {
@@ -190,7 +252,18 @@ replica::replica(replica_stub *stub,
METRIC_VAR_INIT_replica(splitting_rejected_write_requests),
METRIC_VAR_INIT_replica(splitting_rejected_read_requests),
METRIC_VAR_INIT_replica(bulk_load_ingestion_rejected_write_requests),
- METRIC_VAR_INIT_replica(dup_rejected_non_idempotent_write_requests)
+ METRIC_VAR_INIT_replica(dup_rejected_non_idempotent_write_requests),
+ METRIC_VAR_INIT_replica(learn_count),
+ METRIC_VAR_INIT_replica(learn_rounds),
+ METRIC_VAR_INIT_replica(learn_copy_files),
+ METRIC_VAR_INIT_replica(learn_copy_file_bytes),
+ METRIC_VAR_INIT_replica(learn_copy_buffer_bytes),
+ METRIC_VAR_INIT_replica(learn_lt_cache_responses),
+ METRIC_VAR_INIT_replica(learn_lt_app_responses),
+ METRIC_VAR_INIT_replica(learn_lt_log_responses),
+ METRIC_VAR_INIT_replica(learn_resets),
+ METRIC_VAR_INIT_replica(learn_failed_count),
+ METRIC_VAR_INIT_replica(learn_successful_count)
{
CHECK(!_app_info.app_type.empty(), "");
CHECK_NOTNULL(stub, "");
diff --git a/src/replica/replica.h b/src/replica/replica.h
index ae00a7d19..bbceeb4b6 100644
--- a/src/replica/replica.h
+++ b/src/replica/replica.h
@@ -665,6 +665,18 @@ private:
METRIC_VAR_DECLARE_counter(dup_rejected_non_idempotent_write_requests);
std::vector<perf_counter *> _counters_table_level_latency;
+ METRIC_VAR_DECLARE_counter(learn_count);
+ METRIC_VAR_DECLARE_counter(learn_rounds);
+ METRIC_VAR_DECLARE_counter(learn_copy_files);
+ METRIC_VAR_DECLARE_counter(learn_copy_file_bytes);
+ METRIC_VAR_DECLARE_counter(learn_copy_buffer_bytes);
+ METRIC_VAR_DECLARE_counter(learn_lt_cache_responses);
+ METRIC_VAR_DECLARE_counter(learn_lt_app_responses);
+ METRIC_VAR_DECLARE_counter(learn_lt_log_responses);
+ METRIC_VAR_DECLARE_counter(learn_resets);
+ METRIC_VAR_DECLARE_counter(learn_failed_count);
+ METRIC_VAR_DECLARE_counter(learn_successful_count);
+
dsn::task_tracker _tracker;
// the thread access checker
dsn::thread_access_checker _checker;
diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp
index d3bbfe638..ef355f7e9 100644
--- a/src/replica/replica_learn.cpp
+++ b/src/replica/replica_learn.cpp
@@ -57,8 +57,6 @@
#include "mutation.h"
#include "mutation_log.h"
#include "nfs/nfs_node.h"
-#include "perf_counter/perf_counter.h"
-#include "perf_counter/perf_counter_wrapper.h"
#include "replica.h"
#include "replica/duplication/replica_duplicator_manager.h"
#include "replica/prepare_list.h"
@@ -79,8 +77,21 @@
#include "utils/filesystem.h"
#include "utils/flags.h"
#include "utils/fmt_logging.h"
+#include "utils/metrics.h"
#include "utils/thread_access_checker.h"
+METRIC_DECLARE_counter(learn_count);
+METRIC_DECLARE_counter(learn_rounds);
+METRIC_DECLARE_counter(learn_copy_files);
+METRIC_DECLARE_counter(learn_copy_file_bytes);
+METRIC_DECLARE_counter(learn_copy_buffer_bytes);
+METRIC_DECLARE_counter(learn_lt_cache_responses);
+METRIC_DECLARE_counter(learn_lt_app_responses);
+METRIC_DECLARE_counter(learn_lt_log_responses);
+METRIC_DECLARE_counter(learn_resets);
+METRIC_DECLARE_counter(learn_failed_count);
+METRIC_DECLARE_counter(learn_successful_count);
+
namespace dsn {
namespace replication {
@@ -136,7 +147,7 @@ void replica::init_learn(uint64_t signature)
return;
}
- _stub->_counter_replicas_learning_recent_start_count->increment();
+ METRIC_VAR_INCREMENT(learn_count);
_potential_secondary_states.learning_version = signature;
_potential_secondary_states.learning_start_ts_ns = dsn_now_ns();
@@ -176,7 +187,7 @@ void replica::init_learn(uint64_t signature)
// missed ones need to be loaded via private logs
else {
-
_stub->_counter_replicas_learning_recent_round_start_count->increment();
+ METRIC_VAR_INCREMENT(learn_rounds);
_potential_secondary_states.learning_round_is_running
= true;
_potential_secondary_states.catchup_with_private_log_task =
tasking::create_task(LPC_CATCHUP_WITH_PRIVATE_LOGS,
@@ -230,7 +241,7 @@ void replica::init_learn(uint64_t signature)
return;
}
- _stub->_counter_replicas_learning_recent_round_start_count->increment();
+ METRIC_VAR_INCREMENT(learn_rounds);
_potential_secondary_states.learning_round_is_running = true;
learn_request request;
@@ -601,7 +612,7 @@ void replica::on_learn_reply(error_code err, learn_request
&&req, learn_response
enum_to_string(_potential_secondary_states.learning_status));
_potential_secondary_states.learning_copy_buffer_size +=
resp.state.meta.length();
-
_stub->_counter_replicas_learning_recent_copy_buffer_size->add(resp.state.meta.length());
+ METRIC_VAR_INCREMENT_BY(learn_copy_buffer_bytes, resp.state.meta.length());
if (resp.err != ERR_OK) {
if (resp.err == ERR_INACTIVE_STATE || resp.err ==
ERR_INCONSISTENT_STATE) {
@@ -649,7 +660,7 @@ void replica::on_learn_reply(error_code err, learn_request
&&req, learn_response
_app->last_committed_decree(),
resp.last_committed_decree);
-
_stub->_counter_replicas_learning_recent_learn_reset_count->increment();
+ METRIC_VAR_INCREMENT(learn_resets);
// close app
auto err = _app->close(true);
@@ -742,13 +753,13 @@ void replica::on_learn_reply(error_code err,
learn_request &&req, learn_response
switch (resp.type) {
case learn_type::LT_CACHE:
-
_stub->_counter_replicas_learning_recent_learn_cache_count->increment();
+ METRIC_VAR_INCREMENT(learn_lt_cache_responses);
break;
case learn_type::LT_APP:
- _stub->_counter_replicas_learning_recent_learn_app_count->increment();
+ METRIC_VAR_INCREMENT(learn_lt_app_responses);
break;
case learn_type::LT_LOG:
- _stub->_counter_replicas_learning_recent_learn_log_count->increment();
+ METRIC_VAR_INCREMENT(learn_lt_log_responses);
break;
default:
// do nothing
@@ -1042,8 +1053,8 @@ void replica::on_copy_remote_state_completed(error_code
err,
if (err == ERR_OK) {
_potential_secondary_states.learning_copy_file_count +=
resp.state.files.size();
_potential_secondary_states.learning_copy_file_size += size;
-
_stub->_counter_replicas_learning_recent_copy_file_count->add(resp.state.files.size());
- _stub->_counter_replicas_learning_recent_copy_file_size->add(size);
+ METRIC_VAR_INCREMENT_BY(learn_copy_files, resp.state.files.size());
+ METRIC_VAR_INCREMENT_BY(learn_copy_file_bytes, size);
}
if (err != ERR_OK) {
@@ -1240,7 +1251,7 @@ void replica::handle_learning_error(error_code err, bool
is_local_error)
_data_corrupted = true;
}
- _stub->_counter_replicas_learning_recent_learn_fail_count->increment();
+ METRIC_VAR_INCREMENT(learn_failed_count);
update_local_configuration_with_no_ballot_change(
is_local_error ? partition_status::PS_ERROR :
partition_status::PS_INACTIVE);
@@ -1404,7 +1415,7 @@ void
replica::on_learn_completion_notification_reply(error_code err,
handle_learning_error(resp.err, false);
}
} else {
- _stub->_counter_replicas_learning_recent_learn_succ_count->increment();
+ METRIC_VAR_INCREMENT(learn_successful_count);
}
}
diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp
index c9d1c8ad5..d76f7613f 100644
--- a/src/replica/replica_stub.cpp
+++ b/src/replica/replica_stub.cpp
@@ -110,6 +110,22 @@ METRIC_DEFINE_gauge_int64(server,
dsn::metric_unit::kReplicas,
"The number of closing replicas");
+METRIC_DEFINE_gauge_int64(server,
+ learning_replicas,
+ dsn::metric_unit::kReplicas,
+ "The number of learning replicas");
+
+METRIC_DEFINE_gauge_int64(server,
+ learning_replicas_max_duration_ms,
+ dsn::metric_unit::kMilliSeconds,
+ "The max duration among all learning replicas");
+
+METRIC_DEFINE_gauge_int64(
+ server,
+ learning_replicas_max_copy_file_bytes,
+ dsn::metric_unit::kBytes,
+ "The max size of files that are copied from learnee among all learning
replicas");
+
namespace dsn {
namespace replication {
@@ -215,7 +231,10 @@ replica_stub::replica_stub(replica_state_subscriber
subscriber /*= nullptr*/,
_is_running(false),
METRIC_VAR_INIT_server(total_replicas),
METRIC_VAR_INIT_server(opening_replicas),
- METRIC_VAR_INIT_server(closing_replicas)
+ METRIC_VAR_INIT_server(closing_replicas),
+ METRIC_VAR_INIT_server(learning_replicas),
+ METRIC_VAR_INIT_server(learning_replicas_max_duration_ms),
+ METRIC_VAR_INIT_server(learning_replicas_max_copy_file_bytes)
{
#ifdef DSN_ENABLE_GPERF
_is_releasing_memory = false;
@@ -233,77 +252,6 @@ replica_stub::~replica_stub(void) { close(); }
void replica_stub::install_perf_counters()
{
- _counter_replicas_learning_count.init_app_counter("eon.replica_stub",
-
"replicas.learning.count",
- COUNTER_TYPE_NUMBER,
- "current learning
count");
- _counter_replicas_learning_max_duration_time_ms.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.max.duration.time(ms)",
- COUNTER_TYPE_NUMBER,
- "current learning max duration time(ms)");
- _counter_replicas_learning_max_copy_file_size.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.max.copy.file.size",
- COUNTER_TYPE_NUMBER,
- "current learning max copy file size");
- _counter_replicas_learning_recent_start_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.start.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "current learning start count in the recent period");
- _counter_replicas_learning_recent_round_start_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.round.start.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning round start count in the recent period");
- _counter_replicas_learning_recent_copy_file_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.copy.file.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning copy file count in the recent period");
- _counter_replicas_learning_recent_copy_file_size.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.copy.file.size",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning copy file size in the recent period");
- _counter_replicas_learning_recent_copy_buffer_size.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.copy.buffer.size",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning copy buffer size in the recent period");
- _counter_replicas_learning_recent_learn_cache_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.cache.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning LT_CACHE count in the recent period");
- _counter_replicas_learning_recent_learn_app_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.app.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning LT_APP count in the recent period");
- _counter_replicas_learning_recent_learn_log_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.log.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning LT_LOG count in the recent period");
- _counter_replicas_learning_recent_learn_reset_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.reset.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning reset count in the recent period"
- "for the reason of resp.last_committed_decree <
_app->last_committed_decree()");
- _counter_replicas_learning_recent_learn_fail_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.fail.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning fail count in the recent period");
- _counter_replicas_learning_recent_learn_succ_count.init_app_counter(
- "eon.replica_stub",
- "replicas.learning.recent.learn.succ.count",
- COUNTER_TYPE_VOLATILE_NUMBER,
- "learning succeed count in the recent period");
-
_counter_replicas_recent_prepare_fail_count.init_app_counter(
"eon.replica_stub",
"replicas.recent.prepare.fail.count",
@@ -1972,9 +1920,9 @@ void replica_stub::on_gc()
}
}
- _counter_replicas_learning_count->set(learning_count);
-
_counter_replicas_learning_max_duration_time_ms->set(learning_max_duration_time_ms);
-
_counter_replicas_learning_max_copy_file_size->set(learning_max_copy_file_size);
+ METRIC_VAR_SET(learning_replicas, learning_count);
+ METRIC_VAR_SET(learning_replicas_max_duration_ms,
learning_max_duration_time_ms);
+ METRIC_VAR_SET(learning_replicas_max_copy_file_bytes,
learning_max_copy_file_size);
_counter_cold_backup_running_count->set(cold_backup_running_count);
_counter_cold_backup_max_duration_time_ms->set(cold_backup_max_duration_time_ms);
_counter_cold_backup_max_upload_file_size->set(cold_backup_max_upload_file_size);
diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h
index bae225ef3..67b8ae8d7 100644
--- a/src/replica/replica_stub.h
+++ b/src/replica/replica_stub.h
@@ -472,20 +472,9 @@ private:
METRIC_VAR_DECLARE_gauge_int64(opening_replicas);
METRIC_VAR_DECLARE_gauge_int64(closing_replicas);
- perf_counter_wrapper _counter_replicas_learning_count;
- perf_counter_wrapper _counter_replicas_learning_max_duration_time_ms;
- perf_counter_wrapper _counter_replicas_learning_max_copy_file_size;
- perf_counter_wrapper _counter_replicas_learning_recent_start_count;
- perf_counter_wrapper _counter_replicas_learning_recent_round_start_count;
- perf_counter_wrapper _counter_replicas_learning_recent_copy_file_count;
- perf_counter_wrapper _counter_replicas_learning_recent_copy_file_size;
- perf_counter_wrapper _counter_replicas_learning_recent_copy_buffer_size;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_cache_count;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_app_count;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_log_count;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_reset_count;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_fail_count;
- perf_counter_wrapper _counter_replicas_learning_recent_learn_succ_count;
+ METRIC_VAR_DECLARE_gauge_int64(learning_replicas);
+ METRIC_VAR_DECLARE_gauge_int64(learning_replicas_max_duration_ms);
+ METRIC_VAR_DECLARE_gauge_int64(learning_replicas_max_copy_file_bytes);
perf_counter_wrapper _counter_replicas_recent_prepare_fail_count;
perf_counter_wrapper _counter_replicas_recent_replica_move_error_count;
diff --git a/src/utils/metrics.h b/src/utils/metrics.h
index f9ab9c2dc..b5e31c050 100644
--- a/src/utils/metrics.h
+++ b/src/utils/metrics.h
@@ -656,6 +656,7 @@ enum class metric_unit : size_t
kReplicas,
kServers,
kRequests,
+ kResponses,
kSeeks,
kPointLookups,
kValues,
@@ -669,6 +670,9 @@ enum class metric_unit : size_t
kOperations,
kTasks,
kDisconnections,
+ kLearns,
+ kRounds,
+ kResets,
kInvalidUnit,
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
