This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f043785df6c branch-3.1: [refactor](filecache) clean file cache profile
code #53733 (#58070)
f043785df6c is described below
commit f043785df6cfa0d7cea6a65a02218c499d6bd86c
Author: zhengyu <[email protected]>
AuthorDate: Tue Nov 25 11:13:39 2025 +0800
branch-3.1: [refactor](filecache) clean file cache profile code #53733
(#58070)
picked from #53733
---
be/src/io/cache/block_file_cache_profile.cpp | 107 ++++++++++++++++++++-----
be/src/io/cache/block_file_cache_profile.h | 108 ++++----------------------
be/src/io/cache/cached_remote_file_reader.cpp | 2 +-
3 files changed, 105 insertions(+), 112 deletions(-)
diff --git a/be/src/io/cache/block_file_cache_profile.cpp
b/be/src/io/cache/block_file_cache_profile.cpp
index 68d74f85257..74e926359ed 100644
--- a/be/src/io/cache/block_file_cache_profile.cpp
+++ b/be/src/io/cache/block_file_cache_profile.cpp
@@ -25,36 +25,35 @@
namespace doris::io {
-std::shared_ptr<AtomicStatistics> FileCacheProfile::report() {
+std::shared_ptr<AtomicStatistics> FileCacheMetrics::report() {
std::shared_ptr<AtomicStatistics> stats =
std::make_shared<AtomicStatistics>();
std::lock_guard lock(_mtx);
- stats->num_io_bytes_read_from_cache +=
_profile->num_io_bytes_read_from_cache;
- stats->num_io_bytes_read_from_remote +=
_profile->num_io_bytes_read_from_remote;
- stats->num_io_bytes_read_from_peer +=
_profile->num_io_bytes_read_from_peer;
+ stats->num_io_bytes_read_from_cache +=
_statistics->num_io_bytes_read_from_cache;
+ stats->num_io_bytes_read_from_remote +=
_statistics->num_io_bytes_read_from_remote;
+ stats->num_io_bytes_read_from_peer +=
_statistics->num_io_bytes_read_from_peer;
return stats;
}
-void FileCacheProfile::update(FileCacheStatistics* stats) {
- if (_profile == nullptr) {
+void FileCacheMetrics::update(FileCacheStatistics* stats) {
+ if (_statistics == nullptr) {
std::lock_guard<std::mutex> lock(_mtx);
- if (_profile == nullptr) {
- _profile = std::make_shared<AtomicStatistics>();
- _file_cache_metric = std::make_shared<FileCacheMetric>(this);
- _file_cache_metric->register_entity();
+ if (_statistics == nullptr) {
+ _statistics = std::make_shared<AtomicStatistics>();
+ register_entity();
}
}
- _profile->num_io_bytes_read_from_cache += stats->bytes_read_from_local;
- _profile->num_io_bytes_read_from_remote += stats->bytes_read_from_remote;
- _profile->num_io_bytes_read_from_peer += stats->bytes_read_from_peer;
+ _statistics->num_io_bytes_read_from_cache += stats->bytes_read_from_local;
+ _statistics->num_io_bytes_read_from_remote +=
stats->bytes_read_from_remote;
+ _statistics->num_io_bytes_read_from_peer += stats->bytes_read_from_peer;
}
-void FileCacheMetric::register_entity() {
-
DorisMetrics::instance()->server_entity()->register_hook("block_file_cache",
- [this]() {
update_table_metrics(); });
+void FileCacheMetrics::register_entity() {
+ DorisMetrics::instance()->server_entity()->register_hook(
+ "block_file_cache", [this]() { update_metrics_callback(); });
}
-void FileCacheMetric::update_table_metrics() const {
- auto stats = profile->report();
+void FileCacheMetrics::update_metrics_callback() {
+ std::shared_ptr<AtomicStatistics> stats = report();
DorisMetrics::instance()->num_io_bytes_read_from_cache->set_value(
stats->num_io_bytes_read_from_cache);
DorisMetrics::instance()->num_io_bytes_read_from_remote->set_value(
@@ -66,4 +65,76 @@ void FileCacheMetric::update_table_metrics() const {
stats->num_io_bytes_read_from_peer);
}
+FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) {
+ static const char* cache_profile = "FileCache";
+ ADD_TIMER_WITH_LEVEL(profile, cache_profile, 1);
+ num_local_io_total =
+ ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal",
TUnit::UNIT, cache_profile, 1);
+ num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumRemoteIOTotal", TUnit::UNIT,
+ cache_profile, 1);
+ local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LocalIOUseTimer",
cache_profile, 1);
+ remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RemoteIOUseTimer",
cache_profile, 1);
+ write_cache_io_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "WriteCacheIOUseTimer",
cache_profile, 1);
+ bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesWriteIntoCache",
+ TUnit::BYTES,
cache_profile, 1);
+ num_skip_cache_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumSkipCacheIOTotal",
+ TUnit::UNIT,
cache_profile, 1);
+ bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromCache",
+ TUnit::BYTES,
cache_profile, 1);
+ bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromRemote",
+ TUnit::BYTES,
cache_profile, 1);
+ read_cache_file_directly_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "ReadCacheFileDirectlyTimer",
cache_profile, 1);
+ cache_get_or_set_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "CacheGetOrSetTimer",
cache_profile, 1);
+ lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer",
cache_profile, 1);
+ get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer", cache_profile,
1);
+ set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer", cache_profile,
1);
+
+ inverted_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(
+ profile, "InvertedIndexNumLocalIOTotal", TUnit::UNIT,
cache_profile, 1);
+ inverted_index_num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(
+ profile, "InvertedIndexNumRemoteIOTotal", TUnit::UNIT,
cache_profile, 1);
+ inverted_index_bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(
+ profile, "InvertedIndexBytesScannedFromCache", TUnit::BYTES,
cache_profile, 1);
+ inverted_index_bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(
+ profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES,
cache_profile, 1);
+ inverted_index_local_io_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile,
"InvertedIndexLocalIOUseTimer", cache_profile, 1);
+ inverted_index_remote_io_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile,
"InvertedIndexRemoteIOUseTimer", cache_profile, 1);
+ inverted_index_io_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer",
cache_profile, 1);
+}
+
+void FileCacheProfileReporter::update(const FileCacheStatistics* statistics)
const {
+ COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total);
+ COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total);
+ COUNTER_UPDATE(local_io_timer, statistics->local_io_timer);
+ COUNTER_UPDATE(remote_io_timer, statistics->remote_io_timer);
+ COUNTER_UPDATE(write_cache_io_timer, statistics->write_cache_io_timer);
+ COUNTER_UPDATE(bytes_write_into_cache, statistics->bytes_write_into_cache);
+ COUNTER_UPDATE(num_skip_cache_io_total,
statistics->num_skip_cache_io_total);
+ COUNTER_UPDATE(bytes_scanned_from_cache,
statistics->bytes_read_from_local);
+ COUNTER_UPDATE(bytes_scanned_from_remote,
statistics->bytes_read_from_remote);
+ COUNTER_UPDATE(read_cache_file_directly_timer,
statistics->read_cache_file_directly_timer);
+ COUNTER_UPDATE(cache_get_or_set_timer, statistics->cache_get_or_set_timer);
+ COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer);
+ COUNTER_UPDATE(get_timer, statistics->get_timer);
+ COUNTER_UPDATE(set_timer, statistics->set_timer);
+
+ COUNTER_UPDATE(inverted_index_num_local_io_total,
+ statistics->inverted_index_num_local_io_total);
+ COUNTER_UPDATE(inverted_index_num_remote_io_total,
+ statistics->inverted_index_num_remote_io_total);
+ COUNTER_UPDATE(inverted_index_bytes_scanned_from_cache,
+ statistics->inverted_index_bytes_read_from_local);
+ COUNTER_UPDATE(inverted_index_bytes_scanned_from_remote,
+ statistics->inverted_index_bytes_read_from_remote);
+ COUNTER_UPDATE(inverted_index_local_io_timer,
statistics->inverted_index_local_io_timer);
+ COUNTER_UPDATE(inverted_index_remote_io_timer,
statistics->inverted_index_remote_io_timer);
+ COUNTER_UPDATE(inverted_index_io_timer,
statistics->inverted_index_io_timer);
+}
+
} // namespace doris::io
diff --git a/be/src/io/cache/block_file_cache_profile.h
b/be/src/io/cache/block_file_cache_profile.h
index 3e2492da515..caa12f72bc9 100644
--- a/be/src/io/cache/block_file_cache_profile.h
+++ b/be/src/io/cache/block_file_cache_profile.h
@@ -39,38 +39,29 @@ struct AtomicStatistics {
std::atomic<int64_t> num_io_bytes_read_from_remote = 0;
std::atomic<int64_t> num_io_bytes_read_from_peer = 0;
};
-
-struct FileCacheProfile;
-
-struct FileCacheMetric {
- FileCacheMetric(FileCacheProfile* profile) : profile(profile) {}
-
- void register_entity();
- void update_table_metrics() const;
-
- FileCacheMetric& operator=(const FileCacheMetric&) = delete;
- FileCacheMetric(const FileCacheMetric&) = delete;
- FileCacheProfile* profile = nullptr;
-};
-
-struct FileCacheProfile {
- static FileCacheProfile& instance() {
- static FileCacheProfile s_profile;
- return s_profile;
+class FileCacheMetrics {
+public:
+ static FileCacheMetrics& instance() {
+ static FileCacheMetrics s_metrics;
+ return s_metrics;
}
- FileCacheProfile() {
+ FileCacheMetrics() {
FileCacheStatistics stats;
update(&stats);
}
void update(FileCacheStatistics* stats);
+private:
+ std::shared_ptr<AtomicStatistics> report();
+ void register_entity();
+ void update_metrics_callback();
+
+private:
std::mutex _mtx;
// use shared_ptr for concurrent
- std::shared_ptr<AtomicStatistics> _profile;
- std::shared_ptr<FileCacheMetric> _file_cache_metric;
- std::shared_ptr<AtomicStatistics> report();
+ std::shared_ptr<AtomicStatistics> _statistics;
};
struct FileCacheProfileReporter {
@@ -103,77 +94,8 @@ struct FileCacheProfileReporter {
RuntimeProfile::Counter* inverted_index_peer_io_timer = nullptr;
RuntimeProfile::Counter* inverted_index_io_timer = nullptr;
- FileCacheProfileReporter(RuntimeProfile* profile) {
- static const char* cache_profile = "FileCache";
- ADD_TIMER_WITH_LEVEL(profile, cache_profile, 1);
- num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumLocalIOTotal", TUnit::UNIT,
- cache_profile, 1);
- num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumRemoteIOTotal", TUnit::UNIT,
- cache_profile, 1);
- local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile,
"LocalIOUseTimer", cache_profile, 1);
- remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile,
"RemoteIOUseTimer", cache_profile, 1);
- write_cache_io_timer =
- ADD_CHILD_TIMER_WITH_LEVEL(profile, "WriteCacheIOUseTimer",
cache_profile, 1);
- bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesWriteIntoCache",
- TUnit::BYTES,
cache_profile, 1);
- num_skip_cache_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumSkipCacheIOTotal",
- TUnit::UNIT,
cache_profile, 1);
- bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromCache",
- TUnit::BYTES,
cache_profile, 1);
- bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromRemote",
- TUnit::BYTES,
cache_profile, 1);
- read_cache_file_directly_timer =
- ADD_CHILD_TIMER_WITH_LEVEL(profile,
"ReadCacheFileDirectlyTimer", cache_profile, 1);
- cache_get_or_set_timer =
- ADD_CHILD_TIMER_WITH_LEVEL(profile, "CacheGetOrSetTimer",
cache_profile, 1);
- lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer",
cache_profile, 1);
- get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer",
cache_profile, 1);
- set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer",
cache_profile, 1);
-
- inverted_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(
- profile, "InvertedIndexNumLocalIOTotal", TUnit::UNIT,
cache_profile, 1);
- inverted_index_num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(
- profile, "InvertedIndexNumRemoteIOTotal", TUnit::UNIT,
cache_profile, 1);
- inverted_index_bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(
- profile, "InvertedIndexBytesScannedFromCache", TUnit::BYTES,
cache_profile, 1);
- inverted_index_bytes_scanned_from_remote =
ADD_CHILD_COUNTER_WITH_LEVEL(
- profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES,
cache_profile, 1);
- inverted_index_local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(
- profile, "InvertedIndexLocalIOUseTimer", cache_profile, 1);
- inverted_index_remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(
- profile, "InvertedIndexRemoteIOUseTimer", cache_profile, 1);
- inverted_index_io_timer =
- ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer",
cache_profile, 1);
- }
-
- void update(const FileCacheStatistics* statistics) const {
- COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total);
- COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total);
- COUNTER_UPDATE(local_io_timer, statistics->local_io_timer);
- COUNTER_UPDATE(remote_io_timer, statistics->remote_io_timer);
- COUNTER_UPDATE(write_cache_io_timer, statistics->write_cache_io_timer);
- COUNTER_UPDATE(bytes_write_into_cache,
statistics->bytes_write_into_cache);
- COUNTER_UPDATE(num_skip_cache_io_total,
statistics->num_skip_cache_io_total);
- COUNTER_UPDATE(bytes_scanned_from_cache,
statistics->bytes_read_from_local);
- COUNTER_UPDATE(bytes_scanned_from_remote,
statistics->bytes_read_from_remote);
- COUNTER_UPDATE(read_cache_file_directly_timer,
statistics->read_cache_file_directly_timer);
- COUNTER_UPDATE(cache_get_or_set_timer,
statistics->cache_get_or_set_timer);
- COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer);
- COUNTER_UPDATE(get_timer, statistics->get_timer);
- COUNTER_UPDATE(set_timer, statistics->set_timer);
-
- COUNTER_UPDATE(inverted_index_num_local_io_total,
- statistics->inverted_index_num_local_io_total);
- COUNTER_UPDATE(inverted_index_num_remote_io_total,
- statistics->inverted_index_num_remote_io_total);
- COUNTER_UPDATE(inverted_index_bytes_scanned_from_cache,
- statistics->inverted_index_bytes_read_from_local);
- COUNTER_UPDATE(inverted_index_bytes_scanned_from_remote,
- statistics->inverted_index_bytes_read_from_remote);
- COUNTER_UPDATE(inverted_index_local_io_timer,
statistics->inverted_index_local_io_timer);
- COUNTER_UPDATE(inverted_index_remote_io_timer,
statistics->inverted_index_remote_io_timer);
- COUNTER_UPDATE(inverted_index_io_timer,
statistics->inverted_index_io_timer);
- }
+ FileCacheProfileReporter(RuntimeProfile* profile);
+ void update(const FileCacheStatistics* statistics) const;
};
} // namespace io
diff --git a/be/src/io/cache/cached_remote_file_reader.cpp
b/be/src/io/cache/cached_remote_file_reader.cpp
index d79bbbe6e6c..af89c1e162f 100644
--- a/be/src/io/cache/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -320,7 +320,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
// update stats increment in this reading procedure for file cache
metrics
FileCacheStatistics fcache_stats_increment;
_update_stats(stats, &fcache_stats_increment,
io_ctx->is_inverted_index);
- io::FileCacheProfile::instance().update(&fcache_stats_increment);
+ io::FileCacheMetrics::instance().update(&fcache_stats_increment);
}
};
std::unique_ptr<int, decltype(defer_func)> defer((int*)0x01,
std::move(defer_func));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
