This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 2cd324d713a [fix](cloud) fix evict in advance only last for short time
pick#47892 (#47917)
2cd324d713a is described below
commit 2cd324d713a54a49387c7f1fb7e8593f23ec3a1e
Author: zhengyu <[email protected]>
AuthorDate: Fri Feb 14 19:30:49 2025 +0800
[fix](cloud) fix evict in advance only last for short time pick#47892
(#47917)
pick #47892
the condition of evict in advance will be set false if it is treated as
non-overflow. we must force evict even if not overflow.
---
be/src/io/cache/block_file_cache.cpp | 58 ++++++++++++++++++++++--------------
be/src/io/cache/block_file_cache.h | 15 ++++++----
2 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 7c49a61382c..b5a985fa1f1 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -898,9 +898,9 @@ void BlockFileCache::find_evict_candidates(LRUQueue& queue,
size_t size, size_t
size_t& removed_size,
std::vector<FileBlockCell*>&
to_evict,
std::lock_guard<std::mutex>&
cache_lock,
- size_t& cur_removed_size) {
+ size_t& cur_removed_size, bool
evict_in_advance) {
for (const auto& [entry_key, entry_offset, entry_size] : queue) {
- if (!is_overflow(removed_size, size, cur_cache_size)) {
+ if (!is_overflow(removed_size, size, cur_cache_size,
evict_in_advance)) {
break;
}
auto* cell = get_cell(entry_key, entry_offset, cache_lock);
@@ -1034,10 +1034,16 @@ void BlockFileCache::try_evict_in_advance(size_t size,
std::lock_guard<std::mute
UInt128Wrapper hash = UInt128Wrapper();
size_t offset = 0;
CacheContext context;
+ /* we pick NORMAL and TTL cache to evict in advance
+ * we reserve for them but won't acutually give space to them
+ * on the contrary, NORMAL and TTL may sacrifice by LRU evicting themselves
+ * other cache types cannot be exempted because we will evict what they
have stolen before LRU evicting
+ * in summary: all cache types will shrink somewhat, and NORMAL and TTL
shrink the most, to make sure the cache is not full
+ */
context.cache_type = FileCacheType::NORMAL;
- try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock,
false);
+ try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock,
true);
context.cache_type = FileCacheType::TTL;
- try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock,
false);
+ try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock,
true);
}
bool BlockFileCache::remove_if_ttl_file_blocks(const UInt128Wrapper& file_key,
bool remove_directly,
@@ -1197,7 +1203,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper&
hash, size_t offset, size
bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types,
size_t size,
- int64_t cur_time, std::lock_guard<std::mutex>& cache_lock, bool
sync_removal) {
+ int64_t cur_time, std::lock_guard<std::mutex>& cache_lock, bool
evict_in_advance) {
size_t removed_size = 0;
size_t cur_cache_size = _cur_cache_size;
std::vector<FileBlockCell*> to_evict;
@@ -1205,7 +1211,7 @@ bool
BlockFileCache::try_reserve_from_other_queue_by_time_interval(
auto& queue = get_queue(cache_type);
size_t remove_size_per_type = 0;
for (const auto& [entry_key, entry_offset, entry_size] : queue) {
- if (!is_overflow(removed_size, size, cur_cache_size)) {
+ if (!is_overflow(removed_size, size, cur_cache_size,
evict_in_advance)) {
break;
}
auto* cell = get_cell(entry_key, entry_offset, cache_lock);
@@ -1230,14 +1236,19 @@ bool
BlockFileCache::try_reserve_from_other_queue_by_time_interval(
}
*(_evict_by_time_metrics_matrix[cache_type][cur_type]) <<
remove_size_per_type;
}
- remove_file_blocks(to_evict, cache_lock, sync_removal);
+ bool is_sync_removal = !evict_in_advance;
+ remove_file_blocks(to_evict, cache_lock, is_sync_removal);
- return !is_overflow(removed_size, size, cur_cache_size);
+ return !is_overflow(removed_size, size, cur_cache_size, evict_in_advance);
}
-bool BlockFileCache::is_overflow(size_t removed_size, size_t need_size,
- size_t cur_cache_size) const {
+bool BlockFileCache::is_overflow(size_t removed_size, size_t need_size, size_t
cur_cache_size,
+ bool evict_in_advance) const {
bool ret = false;
+ if (evict_in_advance) { // we don't need to check
_need_evict_cache_in_advance
+ ret = (removed_size < need_size);
+ return ret;
+ }
if (_disk_resource_limit_mode) {
ret = (removed_size < need_size);
} else {
@@ -1248,7 +1259,7 @@ bool BlockFileCache::is_overflow(size_t removed_size,
size_t need_size,
bool BlockFileCache::try_reserve_from_other_queue_by_size(
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types,
size_t size,
- std::lock_guard<std::mutex>& cache_lock, bool sync_removal) {
+ std::lock_guard<std::mutex>& cache_lock, bool evict_in_advance) {
size_t removed_size = 0;
size_t cur_cache_size = _cur_cache_size;
std::vector<FileBlockCell*> to_evict;
@@ -1265,21 +1276,22 @@ bool
BlockFileCache::try_reserve_from_other_queue_by_size(
}
size_t cur_removed_size = 0;
find_evict_candidates(queue, size, cur_cache_size, removed_size,
to_evict, cache_lock,
- cur_removed_size);
+ cur_removed_size, evict_in_advance);
*(_evict_by_size_metrics_matrix[cache_type][cur_type]) <<
cur_removed_size;
}
- remove_file_blocks(to_evict, cache_lock, sync_removal);
- return !is_overflow(removed_size, size, cur_cache_size);
+ bool is_sync_removal = !evict_in_advance;
+ remove_file_blocks(to_evict, cache_lock, is_sync_removal);
+ return !is_overflow(removed_size, size, cur_cache_size, evict_in_advance);
}
bool BlockFileCache::try_reserve_from_other_queue(FileCacheType
cur_cache_type, size_t size,
int64_t cur_time,
std::lock_guard<std::mutex>&
cache_lock,
- bool sync_removal) {
+ bool evict_in_advance) {
// currently, TTL cache is not considered as a candidate
auto other_cache_types = get_other_cache_type_without_ttl(cur_cache_type);
bool reserve_success = try_reserve_from_other_queue_by_time_interval(
- cur_cache_type, other_cache_types, size, cur_time, cache_lock,
sync_removal);
+ cur_cache_type, other_cache_types, size, cur_time, cache_lock,
evict_in_advance);
if (reserve_success ||
!config::file_cache_enable_evict_from_other_queue_by_size) {
return reserve_success;
}
@@ -1293,18 +1305,19 @@ bool
BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type,
return false;
}
return try_reserve_from_other_queue_by_size(cur_cache_type,
other_cache_types, size, cache_lock,
- sync_removal);
+ evict_in_advance);
}
bool BlockFileCache::try_reserve_for_lru(const UInt128Wrapper& hash,
QueryFileCacheContextPtr
query_context,
const CacheContext& context, size_t
offset, size_t size,
std::lock_guard<std::mutex>&
cache_lock,
- bool sync_removal) {
+ bool evict_in_advance) {
int64_t cur_time = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
- if (!try_reserve_from_other_queue(context.cache_type, size, cur_time,
cache_lock)) {
+ if (!try_reserve_from_other_queue(context.cache_type, size, cur_time,
cache_lock,
+ evict_in_advance)) {
auto& queue = get_queue(context.cache_type);
size_t removed_size = 0;
size_t cur_cache_size = _cur_cache_size;
@@ -1312,11 +1325,12 @@ bool BlockFileCache::try_reserve_for_lru(const
UInt128Wrapper& hash,
std::vector<FileBlockCell*> to_evict;
size_t cur_removed_size = 0;
find_evict_candidates(queue, size, cur_cache_size, removed_size,
to_evict, cache_lock,
- cur_removed_size);
- remove_file_blocks(to_evict, cache_lock, sync_removal);
+ cur_removed_size, evict_in_advance);
+ bool is_sync_removal = !evict_in_advance;
+ remove_file_blocks(to_evict, cache_lock, is_sync_removal);
*(_evict_by_self_lru_metrics_matrix[context.cache_type]) <<
cur_removed_size;
- if (is_overflow(removed_size, size, cur_cache_size)) {
+ if (is_overflow(removed_size, size, cur_cache_size, evict_in_advance))
{
return false;
}
}
diff --git a/be/src/io/cache/block_file_cache.h
b/be/src/io/cache/block_file_cache.h
index 5b998708241..5634621eb41 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -414,7 +414,8 @@ private:
bool try_reserve_for_lru(const UInt128Wrapper& hash,
QueryFileCacheContextPtr query_context,
const CacheContext& context, size_t offset,
size_t size,
- std::lock_guard<std::mutex>& cache_lock, bool
sync_removal = true);
+ std::lock_guard<std::mutex>& cache_lock,
+ bool evict_in_advance = false);
bool try_reserve_during_async_load(size_t size,
std::lock_guard<std::mutex>& cache_lock);
@@ -423,7 +424,7 @@ private:
bool try_reserve_from_other_queue(FileCacheType cur_cache_type, size_t
offset, int64_t cur_time,
std::lock_guard<std::mutex>& cache_lock,
- bool sync_removal = true);
+ bool evict_in_advance = false);
size_t get_available_cache_size(FileCacheType cache_type) const;
@@ -468,14 +469,15 @@ private:
std::vector<FileCacheType> other_cache_types,
size_t size, int64_t
cur_time,
std::lock_guard<std::mutex>& cache_lock,
- bool sync_removal);
+ bool evict_in_advance);
bool try_reserve_from_other_queue_by_size(FileCacheType cur_type,
std::vector<FileCacheType>
other_cache_types,
size_t size,
std::lock_guard<std::mutex>& cache_lock,
- bool sync_removal);
+ bool evict_in_advance);
- bool is_overflow(size_t removed_size, size_t need_size, size_t
cur_cache_size) const;
+ bool is_overflow(size_t removed_size, size_t need_size, size_t
cur_cache_size,
+ bool evict_in_advance) const;
void remove_file_blocks(std::vector<FileBlockCell*>&,
std::lock_guard<std::mutex>&, bool sync);
@@ -484,7 +486,8 @@ private:
void find_evict_candidates(LRUQueue& queue, size_t size, size_t
cur_cache_size,
size_t& removed_size,
std::vector<FileBlockCell*>& to_evict,
- std::lock_guard<std::mutex>& cache_lock,
size_t& cur_removed_size);
+ std::lock_guard<std::mutex>& cache_lock,
size_t& cur_removed_size,
+ bool evict_in_advance);
// info
std::string _cache_base_path;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
