This is an automated email from the ASF dual-hosted git repository.
wyxxxcat pushed a commit to branch branch-4.1.2-recycler
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1.2-recycler by this
push:
new f3c16f57fbb [fix](recycler) Filtering out packed files to reduces read
amplification (#70601)
f3c16f57fbb is described below
commit f3c16f57fbb8793daab23350169931bd4a8989e5
Author: Yixuan Wang <[email protected]>
AuthorDate: Sun Nov 15 19:50:06 2026 -0800
[fix](recycler) Filtering out packed files to reduces read amplification
(#70601)
Co-authored-by: wangyixuan <[email protected]>
---
cloud/src/recycler/recycler.cpp | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index b2477f3df9b..46c4d0e192a 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -1613,7 +1613,7 @@ int InstanceRecycler::process_single_packed_file(const
std::string& packed_key,
return -1;
}
-int InstanceRecycler::handle_packed_file_kv(std::string_view key,
std::string_view /*value*/,
+int InstanceRecycler::handle_packed_file_kv(std::string_view key,
std::string_view value,
PackedFileRecycleStats* stats,
int* ret) {
if (stats) {
++stats->num_scanned;
@@ -1632,6 +1632,31 @@ int
InstanceRecycler::handle_packed_file_kv(std::string_view key, std::string_vi
return 0;
}
+ cloud::PackedFileInfoPB packed_info;
+ if (!packed_info.ParseFromArray(value.data(), value.size())) {
+ LOG_WARNING("failed to parse packed file info from scan")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ if (stats) {
+ ++stats->num_failed;
+ }
+ if (ret) {
+ *ret = -1;
+ }
+ return 0;
+ }
+
+ const int64_t now_sec = ::time(nullptr);
+ const bool due =
+ config::force_immediate_recycle ||
+ now_sec - packed_info.created_at_sec() >=
config::packed_file_correction_delay_seconds;
+ const bool need_correction = !packed_info.corrected() && due;
+ const bool need_recycle =
+ packed_info.state() == cloud::PackedFileInfoPB::RECYCLING &&
packed_info.ref_cnt() == 0;
+ if (!need_correction && !need_recycle) {
+ return 0;
+ }
+
std::string packed_key(key);
int process_ret = process_single_packed_file(packed_key, packed_file_path,
stats);
if (process_ret != 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]