This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 3edf640914f branch-4.2: [fix](recycler) Filtering out packed files to
reduces read amplification (#70611)
3edf640914f is described below
commit 3edf640914f0ede0ce129e509c18f8c1435c1d1a
Author: Yixuan Wang <[email protected]>
AuthorDate: Fri Nov 20 16:38:34 2026 +0800
branch-4.2: [fix](recycler) Filtering out packed files to reduces read
amplification (#70611)
pick: https://github.com/apache/doris/pull/63295
Co-authored-by: wangyixuan <[email protected]>
---
cloud/src/recycler/recycler.cpp | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 4aeddd4374d..b39888ca9e5 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -1801,7 +1801,7 @@ int InstanceRecycler::process_single_packed_file(const
std::string& packed_key,
return -1;
}
-int InstanceRecycler::handle_packed_file_kv(std::string_view key,
std::string_view /*value*/,
+int InstanceRecycler::handle_packed_file_kv(std::string_view key,
std::string_view value,
PackedFileRecycleStats* stats,
int* ret) {
if (stats) {
++stats->num_scanned;
@@ -1820,6 +1820,31 @@ int
InstanceRecycler::handle_packed_file_kv(std::string_view key, std::string_vi
return 0;
}
+ cloud::PackedFileInfoPB packed_info;
+ if (!packed_info.ParseFromArray(value.data(), value.size())) {
+ LOG_WARNING("failed to parse packed file info from scan")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ if (stats) {
+ ++stats->num_failed;
+ }
+ if (ret) {
+ *ret = -1;
+ }
+ return 0;
+ }
+
+ const int64_t now_sec = ::time(nullptr);
+ const bool due =
+ config::force_immediate_recycle ||
+ now_sec - packed_info.created_at_sec() >=
config::packed_file_correction_delay_seconds;
+ const bool need_correction = !packed_info.corrected() && due;
+ const bool need_recycle =
+ packed_info.state() == cloud::PackedFileInfoPB::RECYCLING &&
packed_info.ref_cnt() == 0;
+ if (!need_correction && !need_recycle) {
+ return 0;
+ }
+
std::string packed_key(key);
int process_ret = process_single_packed_file(packed_key, packed_file_path,
stats);
if (process_ret != 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]