This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c8aaebb49be branch-3.0: [enhance](cloud)add log for compaction read
time (#52168)
c8aaebb49be is described below
commit c8aaebb49be0ace547ea92b95365ad0cd40ffbe6
Author: koarz <[email protected]>
AuthorDate: Wed Jun 25 09:46:08 2025 +0800
branch-3.0: [enhance](cloud)add log for compaction read time (#52168)
https://github.com/apache/doris/pull/51355
---
be/src/cloud/cloud_base_compaction.cpp | 2 ++
be/src/cloud/cloud_cumulative_compaction.cpp | 2 ++
be/src/cloud/cloud_full_compaction.cpp | 2 ++
be/src/cloud/cloud_tablet.cpp | 15 +++++++++++++++
be/src/cloud/cloud_tablet.h | 4 ++++
be/src/olap/compaction.cpp | 6 ++++++
be/src/olap/merger.cpp | 6 ++++++
be/src/olap/merger.h | 2 ++
8 files changed, 39 insertions(+)
diff --git a/be/src/cloud/cloud_base_compaction.cpp
b/be/src/cloud/cloud_base_compaction.cpp
index 2b9fd1c2e56..963ce1fa587 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -301,6 +301,8 @@ Status CloudBaseCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index 8546ac04fcf..39954097324 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -224,6 +224,8 @@ Status CloudCumulativeCompaction::execute_compact() {
.tag("cumulative_point", cloud_tablet()->cumulative_layer_point())
.tag("num_rowsets",
cloud_tablet()->fetch_add_approximate_num_rowsets(0))
.tag("cumu_num_rowsets",
cloud_tablet()->fetch_add_approximate_cumu_num_rowsets(0))
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_full_compaction.cpp
b/be/src/cloud/cloud_full_compaction.cpp
index f04b7d1d4a7..6bfab2ec698 100644
--- a/be/src/cloud/cloud_full_compaction.cpp
+++ b/be/src/cloud/cloud_full_compaction.cpp
@@ -183,6 +183,8 @@ Status CloudFullCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index b4cf9d2e31c..a26ce0dd2c7 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -730,6 +730,21 @@ void CloudTablet::get_compaction_status(std::string*
json_result) {
static_cast<uint>(_last_full_compaction_status.length()),
root.GetAllocator());
root.AddMember("last full status", full_compaction_status_value,
root.GetAllocator());
+ rapidjson::Value exec_compaction_time;
+ std::string num_str {std::to_string(exec_compaction_time_us.load())};
+ exec_compaction_time.SetString(num_str.c_str(),
static_cast<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("exec compaction time us", exec_compaction_time,
root.GetAllocator());
+ rapidjson::Value local_read_time;
+ num_str = std::to_string(local_read_time_us.load());
+ local_read_time.SetString(num_str.c_str(),
static_cast<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("compaction local read time us", local_read_time,
root.GetAllocator());
+ rapidjson::Value remote_read_time;
+ num_str = std::to_string(remote_read_time_us.load());
+ remote_read_time.SetString(num_str.c_str(),
static_cast<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("compaction remote read time us", remote_read_time,
root.GetAllocator());
// print all rowsets' version as an array
rapidjson::Document versions_arr;
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index ee3fdc35c6a..f3af8b09b27 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -251,6 +251,10 @@ public:
int64_t last_cumu_no_suitable_version_ms = 0;
int64_t last_access_time_ms = 0;
+ std::atomic<int64_t> local_read_time_us = 0;
+ std::atomic<int64_t> remote_read_time_us = 0;
+ std::atomic<int64_t> exec_compaction_time_us = 0;
+
// Return merged extended schema
TabletSchemaSPtr merged_tablet_schema() const override;
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index dccf56e9c9b..7e312c5847f 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -1359,6 +1359,12 @@ Status
CloudCompactionMixin::execute_compact_impl(int64_t permits) {
// 4. modify rowsets in memory
RETURN_IF_ERROR(modify_rowsets());
+ // update compaction status data
+ auto tablet = std::static_pointer_cast<CloudTablet>(_tablet);
+ tablet->local_read_time_us.fetch_add(_stats.cloud_local_read_time);
+ tablet->remote_read_time_us.fetch_add(_stats.cloud_remote_read_time);
+ tablet->exec_compaction_time_us.fetch_add(watch.get_elapse_time_us());
+
return Status::OK();
}
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index b213808af24..01f36d77dc2 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -143,6 +143,9 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet,
ReaderType reader_type,
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total =
reader.stats().file_cache_stats.bytes_write_into_cache;
+ stats_output->cloud_local_read_time =
reader.stats().file_cache_stats.local_io_timer / 1000;
+ stats_output->cloud_remote_read_time =
+ reader.stats().file_cache_stats.remote_io_timer / 1000;
}
RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->flush(),
@@ -320,6 +323,9 @@ Status Merger::vertical_compact_one_group(
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total =
reader.stats().file_cache_stats.bytes_write_into_cache;
+ stats_output->cloud_local_read_time =
reader.stats().file_cache_stats.local_io_timer / 1000;
+ stats_output->cloud_remote_read_time =
+ reader.stats().file_cache_stats.remote_io_timer / 1000;
}
RETURN_IF_ERROR(dst_rowset_writer->flush_columns(is_key));
diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h
index 76d053a7a79..302e606371a 100644
--- a/be/src/olap/merger.h
+++ b/be/src/olap/merger.h
@@ -43,6 +43,8 @@ class VerticalBlockReader;
class Merger {
public:
struct Statistics {
+ int64_t cloud_local_read_time = 0;
+ int64_t cloud_remote_read_time = 0;
// number of rows written to the destination rowset after merge
int64_t output_rows = 0;
int64_t merged_rows = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
