This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 6ec0bc9c7eb branch-3.0: [chore](binlog) add ingesting/downloading
binlog latency metrics #48599 (#48648)
6ec0bc9c7eb is described below
commit 6ec0bc9c7eb6c6811624200eb576bc4205e749a2
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Mar 10 20:59:46 2025 +0800
branch-3.0: [chore](binlog) add ingesting/downloading binlog latency
metrics #48599 (#48648)
Cherry-picked from #48599
Co-authored-by: walter <[email protected]>
---
be/src/http/action/download_binlog_action.cpp | 13 +++++++++++++
be/src/service/backend_service.cpp | 6 ++++--
be/src/util/stopwatch.hpp | 6 ++++++
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/be/src/http/action/download_binlog_action.cpp
b/be/src/http/action/download_binlog_action.cpp
index 4bb8b8b70dd..abdc1f9e321 100644
--- a/be/src/http/action/download_binlog_action.cpp
+++ b/be/src/http/action/download_binlog_action.cpp
@@ -34,6 +34,7 @@
#include "olap/storage_engine.h"
#include "olap/tablet_manager.h"
#include "runtime/exec_env.h"
+#include "util/stopwatch.hpp"
namespace doris {
@@ -47,6 +48,12 @@ const std::string kSegmentIndexParameter = "segment_index";
const std::string kSegmentIndexIdParameter = "segment_index_id";
const std::string kAcquireMD5Parameter = "acquire_md5";
+bvar::LatencyRecorder g_get_binlog_info_latency("doris_download_binlog",
"get_binlog_info");
+bvar::LatencyRecorder g_get_segment_file_latency("doris_download_binlog",
"get_segment_file");
+bvar::LatencyRecorder g_get_segment_index_file_latency("doris_download_binlog",
+
"get_segment_index_file");
+bvar::LatencyRecorder g_get_rowset_meta_latency("doris_download_binlog",
"get_rowset_meta");
+
// get http param, if no value throw exception
const auto& get_http_param(HttpRequest* req, const std::string& param_name) {
const auto& param = req->param(param_name);
@@ -233,14 +240,20 @@ void DownloadBinlogAction::handle(HttpRequest* req) {
const std::string& method = req->param(kMethodParameter);
// Step 3: dispatch
+ MonotonicStopWatch watch;
+ watch.start();
if (method == "get_binlog_info") {
handle_get_binlog_info(_engine, req);
+ g_get_binlog_info_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_file") {
handle_get_segment_file(_engine, req, _rate_limit_group.get());
+ g_get_segment_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_index_file") {
handle_get_segment_index_file(_engine, req, _rate_limit_group.get());
+ g_get_segment_index_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_rowset_meta") {
handle_get_rowset_meta(_engine, req);
+ g_get_rowset_meta_latency << watch.elapsed_time_microseconds();
} else {
auto error_msg = fmt::format("invalid method: {}", method);
LOG(WARNING) << error_msg;
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 817f7ffb914..b7cd5532866 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -88,9 +88,10 @@ class TTransportException;
} // namespace apache
namespace doris {
-
namespace {
+bvar::LatencyRecorder g_ingest_binlog_latency("doris_backend_service",
"ingest_binlog");
+
struct IngestBinlogArg {
int64_t txn_id;
int64_t partition_id;
@@ -121,7 +122,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
std::vector<std::string> download_success_files;
Defer defer {[=, &engine, &tstatus, ingest_binlog_tstatus = arg->tstatus,
&watch,
&total_download_bytes, &total_download_files]() {
- auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() /
1000000);
+ g_ingest_binlog_latency << watch.elapsed_time_microseconds();
+ auto elapsed_time_ms =
static_cast<int64_t>(watch.elapsed_time_milliseconds());
double copy_rate = 0.0;
if (elapsed_time_ms > 0) {
copy_rate = total_download_bytes / ((double)elapsed_time_ms) /
1000;
diff --git a/be/src/util/stopwatch.hpp b/be/src/util/stopwatch.hpp
index ef9c71d16e5..9dc3ee74152 100644
--- a/be/src/util/stopwatch.hpp
+++ b/be/src/util/stopwatch.hpp
@@ -77,6 +77,12 @@ public:
(end.tv_nsec - _start.tv_nsec);
}
+ // Return time in microseconds
+ uint64_t elapsed_time_microseconds() const { return elapsed_time() / 1000;
}
+
+ // Return time in milliseconds
+ uint64_t elapsed_time_milliseconds() const { return elapsed_time() / 1000
/ 1000; }
+
// Returns time in nanosecond.
int64_t elapsed_time_seconds(timespec end) const {
if (!_running) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
