This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 51d0dfd58e9 [chore](recycler) log num deleted objects (#39353)
51d0dfd58e9 is described below
commit 51d0dfd58e93a3fec69f8e834c03065f9a66abfc
Author: walter <[email protected]>
AuthorDate: Wed Aug 14 23:22:30 2024 +0800
[chore](recycler) log num deleted objects (#39353)
---
cloud/src/recycler/obj_storage_client.cpp | 14 +++++++++++++-
cloud/src/recycler/recycler.cpp | 12 ++++--------
cloud/src/recycler/sync_executor.h | 6 ++----
3 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/cloud/src/recycler/obj_storage_client.cpp
b/cloud/src/recycler/obj_storage_client.cpp
index 855fa110a4c..3402bb33468 100644
--- a/cloud/src/recycler/obj_storage_client.cpp
+++ b/cloud/src/recycler/obj_storage_client.cpp
@@ -17,9 +17,12 @@
#include "recycler/obj_storage_client.h"
+#include <chrono>
+
#include "cpp/sync_point.h"
#include "recycler/sync_executor.h"
-#include "recycler/util.h"
+
+using namespace std::chrono;
namespace doris::cloud {
@@ -28,6 +31,9 @@ ObjectStorageResponse
ObjStorageClient::delete_objects_recursively_(ObjectStorag
int64_t
expired_time,
size_t
batch_size) {
TEST_SYNC_POINT_CALLBACK("ObjStorageClient::delete_objects_recursively_",
&batch_size);
+ size_t num_deleted_objects = 0;
+ auto start_time = steady_clock::now();
+
auto list_iter = list_objects(path);
ObjectStorageResponse ret;
@@ -42,6 +48,7 @@ ObjectStorageResponse
ObjStorageClient::delete_objects_recursively_(ObjectStorag
continue;
}
+ num_deleted_objects++;
keys.emplace_back(std::move(obj->key));
if (keys.size() < batch_size) {
continue;
@@ -70,6 +77,11 @@ ObjectStorageResponse
ObjStorageClient::delete_objects_recursively_(ObjectStorag
}
}
+ auto elapsed = duration_cast<milliseconds>(steady_clock::now() -
start_time).count();
+ LOG(INFO) << "delete objects under " << path.bucket << "/" << path.key
+ << " finished, ret=" << ret.ret << ", finished=" << finished
+ << ", num_deleted_objects=" << num_deleted_objects << ", cost="
<< elapsed << " ms";
+
ret = finished ? ret : -1;
return ret;
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 0b2267e601d..1ba33957a20 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -1131,7 +1131,6 @@ int InstanceRecycler::recycle_tablets(int64_t table_id,
int64_t index_id, int64_
return {std::string_view(), range_move};
}
++num_recycled;
- LOG_INFO("k is {}, is empty {}", k, k.empty());
return {k, range_move};
});
} else {
@@ -1157,10 +1156,7 @@ int InstanceRecycler::recycle_tablets(int64_t table_id,
int64_t index_id, int64_
}
return true;
}());
- sync_executor.add([k]() mutable -> TabletKeyPair {
- LOG_INFO("k is {}, is empty {}", k, k.empty());
- return {k, true};
- });
+ sync_executor.add([k]() mutable -> TabletKeyPair { return {k,
true}; });
++num_recycled;
}
return 0;
@@ -1433,7 +1429,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) {
std::unique_ptr<int, std::function<void(int*)>>
defer_log_statistics((int*)0x01, [&](int*) {
auto cost = duration<float>(steady_clock::now() - start_time).count();
- LOG_INFO("recycle rowsets finished, cost={}s", cost)
+ LOG_INFO("recycle the rowsets of dropped tablet finished, cost={}s",
cost)
.tag("instance_id", instance_id_)
.tag("tablet_id", tablet_id);
});
@@ -1618,7 +1614,7 @@ int InstanceRecycler::recycle_rowsets() {
// old version `RecycleRowsetPB` may has empty resource_id,
just remove the kv.
LOG(INFO) << "delete the recycle rowset kv that has empty
resource_id, key="
<< hex(k) << " value=" << proto_to_json(rowset);
- rowset_keys.push_back(std::string(k));
+ rowset_keys.emplace_back(k);
return -1;
}
// decode rowset_id
@@ -1664,7 +1660,7 @@ int InstanceRecycler::recycle_rowsets() {
return -1;
}
} else {
- rowset_keys.push_back(std::string(k));
+ rowset_keys.emplace_back(k);
if (rowset_meta->num_segments() > 0) { // Skip empty rowset
rowsets.push_back(std::move(*rowset_meta));
}
diff --git a/cloud/src/recycler/sync_executor.h
b/cloud/src/recycler/sync_executor.h
index d7009a99ed4..c84e5e22467 100644
--- a/cloud/src/recycler/sync_executor.h
+++ b/cloud/src/recycler/sync_executor.h
@@ -23,10 +23,8 @@
#include <glog/logging.h>
#include <future>
-#include <iostream>
#include <string>
-#include "common/logging.h"
#include "common/simple_thread_pool.h"
namespace doris::cloud {
@@ -54,10 +52,10 @@ public:
auto current_time_second = time(nullptr);
current_time.tv_sec = current_time_second + 300;
current_time.tv_nsec = 0;
- auto msg = fmt::format("{} has already taken 5 min", _name_tag);
while (0 != _count.timed_wait(current_time)) {
current_time.tv_sec += 300;
- LOG(WARNING) << msg;
+ LOG(WARNING) << _name_tag << " has already taken 5 min, cost: "
+ << time(nullptr) - current_time_second << " seconds";
}
*finished = !_stop_token;
std::vector<T> res;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
