This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f2731185c9 [fix](memory) fix cache clean thread (#22472)
f2731185c9 is described below
commit f2731185c90e9d616e16b7edc097f79469567756
Author: Xinyi Zou <[email protected]>
AuthorDate: Tue Aug 8 15:38:29 2023 +0800
[fix](memory) fix cache clean thread (#22472)
fix page cache update last visit time.
fix cache clean thread
---
be/src/common/config.cpp | 3 +--
be/src/common/config.h | 4 ++--
be/src/olap/olap_server.cpp | 17 +++++++++--------
be/src/olap/page_cache.cpp | 2 ++
be/src/olap/page_cache.h | 7 ++++++-
be/src/olap/storage_engine.cpp | 7 +------
be/src/olap/storage_engine.h | 6 ++----
7 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c9c1430754..ad7ed6614c 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -267,8 +267,7 @@ DEFINE_mInt64(column_dictionary_key_size_threshold, "0");
DEFINE_mInt64(memory_limitation_per_thread_for_schema_change_bytes,
"2147483648");
DEFINE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes,
"100000000");
-// the clean interval of file descriptor cache and segment cache
-DEFINE_mInt32(cache_clean_interval, "60");
+DEFINE_mInt32(cache_prune_stale_interval, "10");
// the clean interval of tablet lookup cache
DEFINE_mInt32(tablet_lookup_cache_clean_interval, "30");
DEFINE_mInt32(disk_stat_monitor_interval, "5");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 195058c155..416eae2880 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -312,8 +312,8 @@ DECLARE_mInt64(column_dictionary_key_size_threshold);
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);
-// the clean interval of file descriptor cache and segment cache
-DECLARE_mInt32(cache_clean_interval);
+// the prune stale interval of all cache
+DECLARE_mInt32(cache_prune_stale_interval);
// the clean interval of tablet lookup cache
DECLARE_mInt32(tablet_lookup_cache_clean_interval);
DECLARE_mInt32(disk_stat_monitor_interval);
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 1733776099..eea1f37577 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -66,6 +66,7 @@
#include "olap/task/engine_publish_version_task.h"
#include "olap/task/index_builder.h"
#include "runtime/client_cache.h"
+#include "runtime/memory/cache_manager.h"
#include "service/brpc.h"
#include "service/point_query_executor.h"
#include "util/brpc_client_cache.h"
@@ -176,11 +177,11 @@ Status StorageEngine::start_bg_threads() {
[this]() { this->_tablet_path_check_callback(); },
&_tablet_path_check_thread));
LOG(INFO) << "tablet path check thread started";
- // fd cache clean thread
+ // cache clean thread
RETURN_IF_ERROR(Thread::create(
- "StorageEngine", "fd_cache_clean_thread",
- [this]() { this->_fd_cache_clean_callback(); },
&_fd_cache_clean_thread));
- LOG(INFO) << "fd cache clean thread started";
+ "StorageEngine", "cache_clean_thread", [this]() {
this->_cache_clean_callback(); },
+ &_cache_clean_thread));
+ LOG(INFO) << "cache clean thread started";
// path scan and gc thread
if (config::path_gc_check) {
@@ -247,17 +248,17 @@ Status StorageEngine::start_bg_threads() {
return Status::OK();
}
-void StorageEngine::_fd_cache_clean_callback() {
+void StorageEngine::_cache_clean_callback() {
int32_t interval = 600;
while
(!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
- interval = config::cache_clean_interval;
+ interval = config::cache_prune_stale_interval;
if (interval <= 0) {
- LOG(WARNING) << "config of file descriptor clean interval is
illegal: [" << interval
+ LOG(WARNING) << "config of cache clean interval is illegal: [" <<
interval
<< "], force set to 3600 ";
interval = 3600;
}
- _start_clean_cache();
+ CacheManager::instance()->for_each_cache_prune_stale();
}
}
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index 47a2379576..57049bdc6d 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -62,6 +62,7 @@ bool StoragePageCache::lookup(const CacheKey& key,
PageCacheHandle* handle,
return false;
}
*handle = PageCacheHandle(cache, lru_handle);
+ handle->update_last_visit_time();
return true;
}
@@ -80,6 +81,7 @@ void StoragePageCache::insert(const CacheKey& key, DataPage*
data, PageCacheHand
auto cache = _get_page_cache(page_type);
auto lru_handle = cache->insert(key.encode(), data, data->capacity(),
deleter, priority);
*handle = PageCacheHandle(cache, lru_handle);
+ handle->update_last_visit_time();
}
} // namespace doris
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index b065d52d0e..3f76546013 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -37,7 +37,7 @@ namespace doris {
class PageCacheHandle;
template <typename TAllocator>
-class PageBase : private TAllocator, LRUCacheValueBase {
+class PageBase : private TAllocator, public LRUCacheValueBase {
public:
PageBase() : _data(nullptr), _size(0), _capacity(0) {}
@@ -229,6 +229,11 @@ public:
return Slice(cache_value->data(), cache_value->size());
}
+ void update_last_visit_time() {
+ DataPage* cache_value = (DataPage*)_cache->value(_handle);
+ cache_value->last_visit_time = UnixMillis();
+ }
+
private:
Cache* _cache = nullptr;
Cache::Handle* _handle = nullptr;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 727083e8f2..e8560a1b9a 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -68,7 +68,6 @@
#include "olap/tablet_meta_manager.h"
#include "olap/task/engine_task.h"
#include "olap/txn_manager.h"
-#include "runtime/memory/cache_manager.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/stream_load/stream_load_recorder.h"
#include "util/doris_metrics.h"
@@ -554,7 +553,7 @@ void StorageEngine::stop() {
THREAD_JOIN(_unused_rowset_monitor_thread);
THREAD_JOIN(_garbage_sweeper_thread);
THREAD_JOIN(_disk_stat_monitor_thread);
- THREAD_JOIN(_fd_cache_clean_thread);
+ THREAD_JOIN(_cache_clean_thread);
THREAD_JOIN(_tablet_checkpoint_tasks_producer_thread);
THREAD_JOIN(_async_publish_thread);
THREAD_JOIN(_cold_data_compaction_producer_thread);
@@ -617,10 +616,6 @@ void StorageEngine::clear_transaction_task(const
TTransactionId transaction_id,
LOG(INFO) << "finish to clear transaction task. transaction_id=" <<
transaction_id;
}
-void StorageEngine::_start_clean_cache() {
- CacheManager::instance()->for_each_cache_prune_stale();
-}
-
Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) {
Status res = Status::OK();
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 99237a6687..f89d3f87d6 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -273,7 +273,7 @@ private:
void _disk_stat_monitor_thread_callback();
// clean file descriptors cache
- void _fd_cache_clean_callback();
+ void _cache_clean_callback();
// path gc process function
void _path_gc_thread_callback(DataDir* data_dir);
@@ -287,8 +287,6 @@ private:
// parse the default rowset type config to RowsetTypePB
void _parse_default_rowset_type();
- void _start_clean_cache();
-
// Disk status monitoring. Monitoring unused_flag Road King's new
corresponding root_path unused flag,
// When the unused mark is detected, the corresponding table information
is deleted from the memory, and the disk data does not move.
// When the disk status is unusable, but the unused logo is not
_push_tablet_into_submitted_compactiondetected, you need to download it from
root_path
@@ -399,7 +397,7 @@ private:
// thread to produce both base and cumulative compaction tasks
scoped_refptr<Thread> _compaction_tasks_producer_thread;
scoped_refptr<Thread> _update_replica_infos_thread;
- scoped_refptr<Thread> _fd_cache_clean_thread;
+ scoped_refptr<Thread> _cache_clean_thread;
// threads to clean all file descriptor not actively in use
std::vector<scoped_refptr<Thread>> _path_gc_threads;
// threads to scan disk paths
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
