This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 20cdd9f9860 [branch-3.0] Fix local data dir metric missing (#46200)
(#46643)
20cdd9f9860 is described below
commit 20cdd9f98606d1fdf4a40db289d1127abe0495a1
Author: wangbo <[email protected]>
AuthorDate: Thu Jan 9 08:29:01 2025 +0800
[branch-3.0] Fix local data dir metric missing (#46200) (#46643)
pick #46200
---
be/src/io/fs/local_file_reader.cpp | 12 +++++++++---
be/src/olap/olap_common.h | 2 +-
be/src/runtime/workload_group/workload_group.cpp | 2 +-
.../workload_group/workload_group_metrics.cpp | 20 +++++++++++++-------
.../runtime/workload_group/workload_group_metrics.h | 18 ++++++++++--------
5 files changed, 34 insertions(+), 20 deletions(-)
diff --git a/be/src/io/fs/local_file_reader.cpp
b/be/src/io/fs/local_file_reader.cpp
index 4a41fa479d9..b19fedd8389 100644
--- a/be/src/io/fs/local_file_reader.cpp
+++ b/be/src/io/fs/local_file_reader.cpp
@@ -56,6 +56,7 @@ void BeConfDataDirReader::get_data_dir_by_file_path(io::Path*
file_path,
}
if (file_path->string().compare(0, data_dir_info.path.size(),
data_dir_info.path) == 0) {
*data_dir_arg = data_dir_info.path;
+ break;
}
}
}
@@ -69,7 +70,7 @@ void BeConfDataDirReader::init_be_conf_data_dir(
data_dir_info.path = store_paths[i].path;
data_dir_info.storage_medium = store_paths[i].storage_medium;
data_dir_info.data_dir_type = DataDirType::OLAP_DATA_DIR;
- data_dir_info.bvar_name = "local_data_dir_" + std::to_string(i);
+ data_dir_info.metric_name = "local_data_dir_" + std::to_string(i);
be_config_data_dir_list.push_back(data_dir_info);
}
@@ -78,7 +79,7 @@ void BeConfDataDirReader::init_be_conf_data_dir(
data_dir_info.path = spill_store_paths[i].path;
data_dir_info.storage_medium = spill_store_paths[i].storage_medium;
data_dir_info.data_dir_type = doris::DataDirType::SPILL_DISK_DIR;
- data_dir_info.bvar_name = "spill_data_dir_" + std::to_string(i);
+ data_dir_info.metric_name = "spill_data_dir_" + std::to_string(i);
be_config_data_dir_list.push_back(data_dir_info);
}
@@ -87,9 +88,14 @@ void BeConfDataDirReader::init_be_conf_data_dir(
data_dir_info.path = cache_paths[i].path;
data_dir_info.storage_medium = TStorageMedium::REMOTE_CACHE;
data_dir_info.data_dir_type = doris::DataDirType::DATA_CACHE_DIR;
- data_dir_info.bvar_name = "local_cache_dir_" + std::to_string(i);
+ data_dir_info.metric_name = "local_cache_dir_" + std::to_string(i);
be_config_data_dir_list.push_back(data_dir_info);
}
+
+ std::sort(be_config_data_dir_list.begin(), be_config_data_dir_list.end(),
+ [](const DataDirInfo& a, const DataDirInfo& b) {
+ return a.path.length() > b.path.length();
+ });
}
LocalFileReader::LocalFileReader(Path path, size_t file_size, int fd)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 11249bafb1e..7a62f7d87f7 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -73,7 +73,7 @@ struct DataDirInfo {
bool is_used = false; // whether
available mark
TStorageMedium::type storage_medium = TStorageMedium::HDD; // Storage
medium type: SSD|HDD
DataDirType data_dir_type = DataDirType::OLAP_DATA_DIR;
- std::string bvar_name;
+ std::string metric_name;
};
struct PredicateFilterInfo {
int type = 0;
diff --git a/be/src/runtime/workload_group/workload_group.cpp
b/be/src/runtime/workload_group/workload_group.cpp
index 71932ebc4e5..9328a50ca17 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -69,7 +69,7 @@ WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& tg_info)
_remote_scan_bytes_per_second(tg_info.remote_read_bytes_per_second) {
std::vector<DataDirInfo>& data_dir_list =
io::BeConfDataDirReader::be_config_data_dir_list;
for (const auto& data_dir : data_dir_list) {
- _scan_io_throttle_map[data_dir.path] =
std::make_shared<IOThrottle>(data_dir.bvar_name);
+ _scan_io_throttle_map[data_dir.path] =
std::make_shared<IOThrottle>(data_dir.metric_name);
}
_remote_scan_io_throttle = std::make_shared<IOThrottle>();
diff --git a/be/src/runtime/workload_group/workload_group_metrics.cpp
b/be/src/runtime/workload_group/workload_group_metrics.cpp
index 18ff7aa2f4f..f1bdd0d14c2 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.cpp
+++ b/be/src/runtime/workload_group/workload_group_metrics.cpp
@@ -17,6 +17,8 @@
#include "runtime/workload_group/workload_group_metrics.h"
+#include "io/fs/local_file_reader.h"
+#include "olap/olap_common.h"
#include "runtime/workload_group/workload_group.h"
#include "runtime/workload_management/io_throttle.h"
#include "util/doris_metrics.h"
@@ -32,7 +34,7 @@ WorkloadGroupMetrics::~WorkloadGroupMetrics() {
WorkloadGroupMetrics::WorkloadGroupMetrics(WorkloadGroup* wg) {
_entity = DorisMetrics::instance()->metric_registry()->register_entity(
- "workload_group." + wg->name(), {{"name", wg->name()}});
+ "workload_group." + std::to_string(wg->id()), {{"workload_group",
wg->name()}});
_cpu_time_metric = std::make_unique<doris::MetricPrototype>(
doris::MetricType::COUNTER, doris::MetricUnit::SECONDS,
"workload_group_cpu_time_sec");
@@ -56,13 +58,14 @@ WorkloadGroupMetrics::WorkloadGroupMetrics(WorkloadGroup*
wg) {
_remote_scan_bytes_counter =
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
_remote_scan_bytes_metric.get()));
- for (const auto& [key, io_throttle] : wg->_scan_io_throttle_map) {
+ std::vector<DataDirInfo>& data_dir_list =
io::BeConfDataDirReader::be_config_data_dir_list;
+ for (const auto& data_dir : data_dir_list) {
std::unique_ptr<doris::MetricPrototype> metric =
std::make_unique<doris::MetricPrototype>(
doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
- "workload_group_local_scan_bytes_" +
io_throttle->metric_name());
- _local_scan_bytes_counter_map[key] =
-
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(metric.get()));
- _local_scan_bytes_metric_map[key] = std::move(metric);
+ "workload_group_local_scan_bytes_" + data_dir.metric_name);
+ _local_scan_bytes_counter_map.insert(
+ {data_dir.path,
(IntCounter*)(_entity->register_metric<IntCounter>(metric.get()))});
+ _local_scan_bytes_metric_map.insert({data_dir.path,
std::move(metric)});
}
}
@@ -76,7 +79,10 @@ void WorkloadGroupMetrics::update_memory_used_bytes(int64_t
memory_used) {
void WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path,
uint64_t delta_io_bytes) {
_local_scan_bytes_counter->increment(delta_io_bytes);
- _local_scan_bytes_counter_map[path]->increment((int64_t)delta_io_bytes);
+ auto range = _local_scan_bytes_counter_map.equal_range(path);
+ for (auto it = range.first; it != range.second; ++it) {
+ it->second->increment((int64_t)delta_io_bytes);
+ }
}
void WorkloadGroupMetrics::update_remote_scan_io_bytes(uint64_t
delta_io_bytes) {
diff --git a/be/src/runtime/workload_group/workload_group_metrics.h
b/be/src/runtime/workload_group/workload_group_metrics.h
index e68715df249..5e43ec825c7 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.h
+++ b/be/src/runtime/workload_group/workload_group_metrics.h
@@ -21,6 +21,7 @@
#include <map>
#include <memory>
#include <string>
+#include <unordered_map>
namespace doris {
@@ -62,14 +63,15 @@ private:
std::unique_ptr<doris::MetricPrototype> _local_scan_bytes_metric {nullptr};
std::unique_ptr<doris::MetricPrototype> _remote_scan_bytes_metric
{nullptr};
// NOTE: _local_scan_bytes_metric is sum of all disk's IO
- // _local_disk_io_metric is every disk's IO
- std::map<std::string, std::unique_ptr<doris::MetricPrototype>>
_local_scan_bytes_metric_map;
-
- IntAtomicCounter* _cpu_time_counter {nullptr}; //
used for metric
- IntAtomicCounter* _mem_used_bytes_counter {nullptr}; //
used for metric
- IntAtomicCounter* _local_scan_bytes_counter {nullptr}; //
used for metric
- IntAtomicCounter* _remote_scan_bytes_counter {nullptr}; //
used for metric
- std::map<std::string, IntAtomicCounter*> _local_scan_bytes_counter_map; //
used for metric
+ std::unordered_multimap<std::string,
std::unique_ptr<doris::MetricPrototype>>
+ _local_scan_bytes_metric_map;
+
+ IntAtomicCounter* _cpu_time_counter {nullptr}; // used for metric
+ IntAtomicCounter* _mem_used_bytes_counter {nullptr}; // used for metric
+ IntAtomicCounter* _local_scan_bytes_counter {nullptr}; // used for metric
+ IntAtomicCounter* _remote_scan_bytes_counter {nullptr}; // used for metric
+ std::unordered_multimap<std::string, IntAtomicCounter*>
+ _local_scan_bytes_counter_map; // used for metric
std::atomic<uint64_t> _cpu_time_nanos {0};
std::atomic<uint64_t> _last_cpu_time_nanos {0};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
