This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3fa4c5f5e00 [enhance](Hdfs) Add bvar latencyrecorder for HDFS
operation in Recycler (#39236)
3fa4c5f5e00 is described below
commit 3fa4c5f5e00f2ec5a521349605808cadf24c5640
Author: AlexYue <[email protected]>
AuthorDate: Tue Aug 13 10:16:55 2024 +0800
[enhance](Hdfs) Add bvar latencyrecorder for HDFS operation in Recycler
(#39236)
This pr adds several bvar for HDFS operation in Recycler.
The bvars are as follows:
```C++
bvar::LatencyRecorder hdfs_write_latency("hdfs_write");
bvar::LatencyRecorder hdfs_open_latency("hdfs_open");
bvar::LatencyRecorder hdfs_close_latency("hdfs_close");
bvar::LatencyRecorder hdfs_list_dir("hdfs_list_dir");
bvar::LatencyRecorder hdfs_exist_latency("hdfs_exist");
bvar::LatencyRecorder hdfs_delete_latency("hdfs_delete");
```
---
cloud/src/recycler/azure_obj_client.cpp | 2 ++
cloud/src/recycler/hdfs_accessor.cpp | 33 ++++++++++++++++++++++++++++++---
cloud/src/recycler/s3_accessor.h | 9 +--------
cloud/src/recycler/s3_obj_client.cpp | 2 ++
cloud/src/recycler/util.h | 6 ++++++
5 files changed, 41 insertions(+), 11 deletions(-)
diff --git a/cloud/src/recycler/azure_obj_client.cpp
b/cloud/src/recycler/azure_obj_client.cpp
index 6a20bff0950..b50874f1fd3 100644
--- a/cloud/src/recycler/azure_obj_client.cpp
+++ b/cloud/src/recycler/azure_obj_client.cpp
@@ -34,9 +34,11 @@
#include "common/config.h"
#include "common/logging.h"
+#include "common/stopwatch.h"
#include "cpp/s3_rate_limiter.h"
#include "cpp/sync_point.h"
#include "recycler/s3_accessor.h"
+#include "recycler/util.h"
using namespace Azure::Storage::Blobs;
diff --git a/cloud/src/recycler/hdfs_accessor.cpp
b/cloud/src/recycler/hdfs_accessor.cpp
index d97c0a37aab..e5038735f57 100644
--- a/cloud/src/recycler/hdfs_accessor.cpp
+++ b/cloud/src/recycler/hdfs_accessor.cpp
@@ -17,7 +17,12 @@
#include "recycler/hdfs_accessor.h"
+#include <bvar/latency_recorder.h>
#include <gen_cpp/cloud.pb.h>
+
+#include "common/stopwatch.h"
+#include "recycler/util.h"
+
#ifdef USE_HADOOP_HDFS
#include <hadoop_hdfs/hdfs.h> // IWYU pragma: export
#else
@@ -44,6 +49,12 @@ std::string hdfs_error() {
return fmt::format("({}): {}", std::strerror(errno), err_msg ? err_msg :
"");
}
+bvar::LatencyRecorder hdfs_write_latency("hdfs_write");
+bvar::LatencyRecorder hdfs_open_latency("hdfs_open");
+bvar::LatencyRecorder hdfs_close_latency("hdfs_close");
+bvar::LatencyRecorder hdfs_list_dir("hdfs_list_dir");
+bvar::LatencyRecorder hdfs_exist_latency("hdfs_exist");
+bvar::LatencyRecorder hdfs_delete_latency("hdfs_delete");
} // namespace
class HDFSBuilder {
@@ -292,6 +303,7 @@ private:
// Return null if error occured, return emtpy DirEntries if dir is empty
or doesn't exist.
std::optional<DirEntries> list_directory(const char* dir_path) {
int num_entries = 0;
+ SCOPED_BVAR_LATENCY(hdfs_list_dir);
auto* file_infos = hdfsListDirectory(hdfs_.get(), dir_path,
&num_entries);
if (errno != 0 && errno != ENOENT) {
LOG_WARNING("failed to list hdfs directory")
@@ -430,6 +442,7 @@ int HdfsAccessor::delete_file(const std::string&
relative_path) {
// Path exists
auto path = to_fs_path(relative_path);
LOG_INFO("delete object").tag("uri", to_uri(relative_path)); // Audit log
+ SCOPED_BVAR_LATENCY(hdfs_delete_latency);
ret = hdfsDelete(fs_.get(), path.c_str(), 0);
if (ret != 0) {
LOG_WARNING("failed to delete object")
@@ -443,7 +456,11 @@ int HdfsAccessor::delete_file(const std::string&
relative_path) {
int HdfsAccessor::put_file(const std::string& relative_path, const
std::string& content) {
auto path = to_fs_path(relative_path);
- auto* file = hdfsOpenFile(fs_.get(), path.c_str(), O_WRONLY, 0, 0, 0);
+ hdfsFile file;
+ {
+ SCOPED_BVAR_LATENCY(hdfs_open_latency);
+ file = hdfsOpenFile(fs_.get(), path.c_str(), O_WRONLY, 0, 0, 0);
+ }
if (!file) {
LOG_WARNING("failed to create file")
.tag("uri", to_uri(relative_path))
@@ -453,11 +470,16 @@ int HdfsAccessor::put_file(const std::string&
relative_path, const std::string&
std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
[&](int*) {
if (file) {
+ SCOPED_BVAR_LATENCY(hdfs_close_latency);
hdfsCloseFile(fs_.get(), file);
}
});
- int64_t written_bytes = hdfsWrite(fs_.get(), file, content.data(),
content.size());
+ int64_t written_bytes = 0;
+ {
+ SCOPED_BVAR_LATENCY(hdfs_write_latency);
+ written_bytes = hdfsWrite(fs_.get(), file, content.data(),
content.size());
+ }
if (written_bytes < content.size()) {
LOG_WARNING("failed to write file")
.tag("uri", to_uri(relative_path))
@@ -465,7 +487,11 @@ int HdfsAccessor::put_file(const std::string&
relative_path, const std::string&
return -1;
}
- int ret = hdfsCloseFile(fs_.get(), file);
+ int ret = 0;
+ {
+ SCOPED_BVAR_LATENCY(hdfs_close_latency);
+ ret = hdfsCloseFile(fs_.get(), file);
+ }
file = nullptr;
if (ret != 0) {
LOG_WARNING("failed to close file")
@@ -496,6 +522,7 @@ int HdfsAccessor::list_all(std::unique_ptr<ListIterator>*
res) {
int HdfsAccessor::exists(const std::string& relative_path) {
auto path = to_fs_path(relative_path);
+ SCOPED_BVAR_LATENCY(hdfs_exist_latency);
int ret = hdfsExists(fs_.get(), path.c_str());
#ifdef USE_HADOOP_HDFS
// when calling hdfsExists() and return non-zero code,
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index 41adc93f04f..6886ee5e7c5 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -23,8 +23,7 @@
#include <cstdint>
#include <memory>
-#include "common/stopwatch.h"
-#include "recycler/s3_obj_client.h"
+#include "recycler/obj_storage_client.h"
#include "recycler/storage_vault_accessor.h"
namespace Aws::S3 {
@@ -52,12 +51,6 @@ extern bvar::LatencyRecorder s3_get_bucket_version_latency;
extern bvar::LatencyRecorder s3_copy_object_latency;
}; // namespace s3_bvar
-// The time unit is the same with BE: us
-#define SCOPED_BVAR_LATENCY(bvar_item) \
- StopWatch sw; \
- std::unique_ptr<int, std::function<void(int*)>> defer( \
- (int*)0x01, [&](int*) { bvar_item << sw.elapsed_us(); });
-
struct AccessorRateLimiter {
public:
~AccessorRateLimiter() = default;
diff --git a/cloud/src/recycler/s3_obj_client.cpp
b/cloud/src/recycler/s3_obj_client.cpp
index fc0c7e9e901..53fa821c7e5 100644
--- a/cloud/src/recycler/s3_obj_client.cpp
+++ b/cloud/src/recycler/s3_obj_client.cpp
@@ -30,9 +30,11 @@
#include "common/config.h"
#include "common/logging.h"
+#include "common/stopwatch.h"
#include "cpp/s3_rate_limiter.h"
#include "cpp/sync_point.h"
#include "recycler/s3_accessor.h"
+#include "recycler/util.h"
namespace doris::cloud {
diff --git a/cloud/src/recycler/util.h b/cloud/src/recycler/util.h
index b63090062bf..b6d4d3299b5 100644
--- a/cloud/src/recycler/util.h
+++ b/cloud/src/recycler/util.h
@@ -25,6 +25,12 @@
namespace doris::cloud {
+// The time unit is the same with BE: us
+#define SCOPED_BVAR_LATENCY(bvar_item) \
+ StopWatch sw; \
+ std::unique_ptr<int, std::function<void(int*)>> defer( \
+ (int*)0x01, [&](int*) { bvar_item << sw.elapsed_us(); });
+
class TxnKv;
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
