This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6ec35ec5078 [fix](packed-file) use timestamp-only cleanup for packed
slice index (#59369)
6ec35ec5078 is described below
commit 6ec35ec5078658374c5ee96b75b4f51a5de59ba2
Author: Luwei <[email protected]>
AuthorDate: Mon Jan 5 19:22:45 2026 +0800
[fix](packed-file) use timestamp-only cleanup for packed slice index
(#59369)
---
be/src/io/fs/packed_file_manager.cpp | 23 +++--------------------
be/src/io/fs/packed_file_manager.h | 1 +
2 files changed, 4 insertions(+), 20 deletions(-)
diff --git a/be/src/io/fs/packed_file_manager.cpp
b/be/src/io/fs/packed_file_manager.cpp
index a41ba59b142..032d2963a40 100644
--- a/be/src/io/fs/packed_file_manager.cpp
+++ b/be/src/io/fs/packed_file_manager.cpp
@@ -256,6 +256,7 @@ Status PackedFileManager::append_small_file(const
std::string& path, const Slice
location.packed_file_path = active_state->packed_file_path;
location.offset = active_state->current_offset;
location.size = data.get_size();
+ location.create_time = std::time(nullptr);
location.tablet_id = info.tablet_id;
location.rowset_id = info.rowset_id;
location.resource_id = info.resource_id;
@@ -818,30 +819,12 @@ void PackedFileManager::cleanup_expired_data() {
// Clean up expired global index entries
{
- std::unordered_set<std::string> active_packed_files;
- {
- std::lock_guard<std::timed_mutex>
current_lock(_current_packed_file_mutex);
- for (const auto& [resource_id, state] : _current_packed_files) {
- if (state) {
- active_packed_files.insert(state->packed_file_path);
- }
- }
- }
- {
- std::lock_guard<std::mutex> merge_lock(_packed_files_mutex);
- for (const auto& [path, state] : _uploading_packed_files) {
- active_packed_files.insert(path);
- }
- for (const auto& [path, state] : _uploaded_packed_files) {
- active_packed_files.insert(path);
- }
- }
-
std::lock_guard<std::mutex> global_lock(_global_index_mutex);
auto it = _global_slice_locations.begin();
while (it != _global_slice_locations.end()) {
const auto& index = it->second;
- if (active_packed_files.find(index.packed_file_path) ==
active_packed_files.end()) {
+ if (index.create_time > 0 &&
+ current_time - index.create_time >
config::uploaded_file_retention_seconds) {
it = _global_slice_locations.erase(it);
} else {
++it;
diff --git a/be/src/io/fs/packed_file_manager.h
b/be/src/io/fs/packed_file_manager.h
index beef51d0603..bfa83a240e0 100644
--- a/be/src/io/fs/packed_file_manager.h
+++ b/be/src/io/fs/packed_file_manager.h
@@ -45,6 +45,7 @@ struct PackedSliceLocation {
std::string packed_file_path;
int64_t offset;
int64_t size;
+ int64_t create_time = 0;
int64_t tablet_id = 0;
std::string rowset_id;
std::string resource_id;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
