This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new eacaa4b5926 [enhancement](s3)Add TotalGetRequestTime profile metric to
the S3 reader. (#57519)
eacaa4b5926 is described below
commit eacaa4b59269236cc1b9debc3d08f631eba5a49a
Author: daidai <[email protected]>
AuthorDate: Fri Oct 31 19:47:19 2025 +0800
[enhancement](s3)Add TotalGetRequestTime profile metric to the S3 reader.
(#57519)
### What problem does this PR solve?
Problem Summary:
This PR adds a `TotalGetRequestTime` profile metric to the S3 reader.
The new metric records the total time spent on all `GetObject` requests,
which will be useful for future performance analysis and optimization.
Example profile (Some less relevant information was manually removed.):
```
FILE_SCAN_OPERATOR (id=0. nereids_id=104. table name = lineitem):
CommonCounters:
- ExecTime: 11sec669ms
- OpenTime: 7.395ms
- WaitForDependency[FILE_SCAN_OPERATOR_DEPENDENCY]Time:
11sec634ms
....
CustomCounters:
....
Scanner:
....
- FileReadTime: 4min8sec
- FileScannerGetBlockTime: 4min9sec
- MergedSmallIO: 0ns
- ReadTime: 4min8sec
- RequestBytes: 2.05 MB
- RequestIO: 20.0K (20000)
- ParquetReader: 0ns
- ColumnReadTime: 4min9sec
- FileOpenTime: 365.293ms
- ParseFooterTime: 5min6sec
- ParseMetaTime: 126.383ms
.....
- S3Profile: 0ns
- TooManyRequestErr: 0
- TooManyRequestSleepTime: 0ms
- TotalBytesRead: 199.14 MB
- TotalGetRequestTime: 9min12sec
- TotalGetRequest: 40.0K (40000)
- ScannerCpuTime: 20sec946ms
- ScannerGetBlockTime: 9min19sec
```
---
be/src/io/fs/s3_file_reader.cpp | 4 ++++
be/src/io/fs/s3_file_reader.h | 1 +
2 files changed, 5 insertions(+)
diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp
index 746f4dfb4fa..710ec428b5b 100644
--- a/be/src/io/fs/s3_file_reader.cpp
+++ b/be/src/io/fs/s3_file_reader.cpp
@@ -149,6 +149,7 @@ Status S3FileReader::read_at_impl(size_t offset, Slice
result, size_t* bytes_rea
.count();
s3_file_reader_latency << (end_ts - begin_ts);
}};
+ SCOPED_RAW_TIMER(&_s3_stats.total_get_request_time_ns);
int total_sleep_time = 0;
while (retry_count <= max_retries) {
@@ -215,11 +216,14 @@ void S3FileReader::_collect_profile_before_close() {
_profile, "TooManyRequestSleepTime", TUnit::TIME_MS,
s3_profile_name);
RuntimeProfile::Counter* total_bytes_read =
ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES,
s3_profile_name);
+ RuntimeProfile::Counter* total_get_request_time_ns =
+ ADD_CHILD_TIMER(_profile, "TotalGetRequestTime",
s3_profile_name);
COUNTER_UPDATE(total_get_request_counter,
_s3_stats.total_get_request_counter);
COUNTER_UPDATE(too_many_request_err_counter,
_s3_stats.too_many_request_err_counter);
COUNTER_UPDATE(too_many_request_sleep_time,
_s3_stats.too_many_request_sleep_time_ms);
COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read);
+ COUNTER_UPDATE(total_get_request_time_ns,
_s3_stats.total_get_request_time_ns);
}
}
diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h
index 36fe67b342c..58294ec1891 100644
--- a/be/src/io/fs/s3_file_reader.h
+++ b/be/src/io/fs/s3_file_reader.h
@@ -65,6 +65,7 @@ private:
int64_t too_many_request_err_counter = 0;
int64_t too_many_request_sleep_time_ms = 0;
int64_t total_bytes_read = 0;
+ int64_t total_get_request_time_ns = 0;
};
Path _path;
size_t _file_size;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
