This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a5c8ed1cde0 [branch2.1][fix](cache) Catch the directory_iterator's
error_code (#39922)
a5c8ed1cde0 is described below
commit a5c8ed1cde08c1cb16b9544839033da5bd21635b
Author: Socrates <[email protected]>
AuthorDate: Tue Aug 27 08:00:52 2024 +0800
[branch2.1][fix](cache) Catch the directory_iterator's error_code (#39922)
## Proposed changes
Catch the directory_iterator's error_code to avoid exceptions causing
core dump
---
be/src/io/cache/block/block_lru_file_cache.cpp | 48 ++++++++++++++++++++------
1 file changed, 38 insertions(+), 10 deletions(-)
diff --git a/be/src/io/cache/block/block_lru_file_cache.cpp
b/be/src/io/cache/block/block_lru_file_cache.cpp
index fa4d42581c6..302891ad770 100644
--- a/be/src/io/cache/block/block_lru_file_cache.cpp
+++ b/be/src/io/cache/block/block_lru_file_cache.cpp
@@ -831,7 +831,12 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
/// version 2.0: cache_base_path / key_prefix / key / offset
if (USE_CACHE_VERSION2 && read_file_cache_version() != "2.0") {
// move directories format as version 2.0
- fs::directory_iterator key_it {_cache_base_path};
+ std::error_code ec;
+ fs::directory_iterator key_it {_cache_base_path, ec};
+ if (ec) {
+ return Status::InternalError("Failed to list dir {}: {}",
_cache_base_path,
+ ec.message());
+ }
for (; key_it != fs::directory_iterator(); ++key_it) {
if (key_it->is_directory()) {
std::string cache_key = key_it->path().filename().native();
@@ -874,7 +879,13 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
vectorized::unhex_uint<uint128_t>(key_it->path().filename().native().c_str()));
CacheContext context;
context.query_id = TUniqueId();
- fs::directory_iterator offset_it {key_it->path()};
+ std::error_code ec;
+ fs::directory_iterator offset_it {key_it->path(), ec};
+ if (ec) [[unlikely]] {
+ LOG(WARNING) << "filesystem error, failed to iterate
directory, file="
+ << key_it->path() << " error=" << ec.message();
+ continue;
+ }
for (; offset_it != fs::directory_iterator(); ++offset_it) {
auto offset_with_suffix =
offset_it->path().filename().native();
auto delim_pos = offset_with_suffix.find('_');
@@ -888,7 +899,6 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
std::string suffix =
offset_with_suffix.substr(delim_pos + 1);
// not need persistent any more
if (suffix == "persistent") {
- std::error_code ec;
std::filesystem::remove(offset_it->path(), ec);
if (ec) {
st = Status::IOError(ec.message());
@@ -907,10 +917,13 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
st = Status::IOError("Unexpected file: {}",
offset_it->path().native());
break;
}
+ size = offset_it->file_size(ec);
+ if (ec) [[unlikely]] {
+ st = Status::IOError(ec.message());
+ break;
+ }
- size = offset_it->file_size();
if (size == 0) {
- std::error_code ec;
fs::remove(offset_it->path(), ec);
if (ec) {
LOG(WARNING) << ec.message();
@@ -922,7 +935,6 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
add_cell(key, context, offset, size,
FileBlock::State::DOWNLOADED, cache_lock);
queue_entries.emplace_back(key, offset);
} else {
- std::error_code ec;
std::filesystem::remove(offset_it->path(), ec);
if (ec) {
st = Status::IOError(ec.message());
@@ -938,8 +950,13 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
}
};
+ std::error_code ec;
if constexpr (USE_CACHE_VERSION2) {
- fs::directory_iterator key_prefix_it {_cache_base_path};
+ fs::directory_iterator key_prefix_it {_cache_base_path, ec};
+ if (ec) [[unlikely]] {
+ return Status::InternalError("Failed to list dir {}: {}",
_cache_base_path,
+ ec.message());
+ }
for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it) {
if (!key_prefix_it->is_directory()) {
// maybe version hits file
@@ -948,14 +965,25 @@ Status
LRUFileCache::load_cache_info_into_memory(std::lock_guard<std::mutex>& ca
if (key_prefix_it->path().filename().native().size() !=
KEY_PREFIX_LENGTH) {
LOG(WARNING) << "Unknown directory " <<
key_prefix_it->path().native()
<< ", try to remove it";
- std::filesystem::remove(key_prefix_it->path());
+ std::error_code ec;
+ std::filesystem::remove(key_prefix_it->path(), ec);
+ if (ec) {
+ LOG(WARNING) << "filesystem error, failed to remove file,
file="
+ << key_prefix_it->path() << " error=" <<
ec.message();
+ }
continue;
}
- fs::directory_iterator key_it {key_prefix_it->path()};
+ fs::directory_iterator key_it {key_prefix_it->path(), ec};
+ if (ec) [[unlikely]] {
+ return Status::IOError(ec.message());
+ }
scan_file_cache(key_it);
}
} else {
- fs::directory_iterator key_it {_cache_base_path};
+ fs::directory_iterator key_it {_cache_base_path, ec};
+ if (ec) [[unlikely]] {
+ return Status::IOError(ec.message());
+ }
scan_file_cache(key_it);
}
if (!st) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
