morningman commented on a change in pull request #349: Improve the Backend's
disk info report performance
URL: https://github.com/apache/incubator-doris/pull/349#discussion_r236072107
##########
File path: be/src/olap/olap_engine.cpp
##########
@@ -529,20 +529,60 @@ OLAPStatus
OLAPEngine::get_all_root_path_info(vector<RootPathInfo>* root_paths_i
OLAPStatus res = OLAP_SUCCESS;
root_paths_info->clear();
- std::lock_guard<std::mutex> l(_store_lock);
- for (auto& it : _store_map) {
- root_paths_info->emplace_back(it.second->to_root_path_info());
+ MonotonicStopWatch timer;
+ timer.start();
+ int tablet_counter = 0;
+
+ // get all root path info and construct a path map.
+ // path -> RootPathInfo
+ std::map<std::string, RootPathInfo> path_map;
+ {
+ std::lock_guard<std::mutex> l(_store_lock);
+ for (auto& it : _store_map) {
+ std::string path = it.first;
+ path_map.emplace(path, it.second->to_root_path_info());
+ // if this path is not used, init it's info
+ if (!path_map[path].is_used) {
+ path_map[path].capacity = 1;
+ path_map[path].data_used_capacity = 0;
+ path_map[path].available = 0;
+ }
+ }
+ }
+
+ // for each tablet, get it's data size, and accumulate the path
'data_used_capacity'
+ // which the tablet belongs to.
+ _tablet_map_lock.rdlock();
+ for (auto& entry : _tablet_map) {
+ TableInstances& instance = entry.second;
+ for (auto& tablet : instance.table_arr) {
+ ++tablet_counter;
+ int64_t data_size = tablet->get_data_size();
+ auto find = path_map.find(tablet->storage_root_path_name());
+ if (find == path_map.end()) {
+ continue;
+ }
+ if (find->second.is_used) {
+ find->second.data_used_capacity += data_size;
+ }
+ }
+ }
+ _tablet_map_lock.unlock();
+
+ // add path info to root_paths_info
+ for (auto& entry : path_map) {
+ root_paths_info->emplace_back(entry.second);
}
+ // get available capacity of each path
for (auto& info: *root_paths_info) {
if (info.is_used) {
- _get_root_path_capacity(info.path, &info.data_used_capacity,
&info.available);
- } else {
- info.capacity = 1;
- info.data_used_capacity = 0;
- info.available = 0;
+ _get_path_available_capacity(info.path, &info.available);
Review comment:
For now, available capacity does not equal to the `capacity -
data_used_capacity`;
Actually, capacity = data_used_capacity + available_capacity +
other_used_capacity.
`other_used_capacity` includes logs/, trash/, mini_download/, meta/, etc,
which are not included in `data_used_capacity`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
