This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 85720ac67eb branch-3.0: [Fix](sk) All sk in log should be encrypted
(#43544) (#43802)
85720ac67eb is described below
commit 85720ac67eb116d705cb0ce878934cd1080cdd1a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Nov 13 15:39:12 2024 +0800
branch-3.0: [Fix](sk) All sk in log should be encrypted (#43544) (#43802)
Cherry-picked from #43544
Co-authored-by: abmdocrt <[email protected]>
---
cloud/src/meta-service/meta_service_helper.h | 45 +++++++++++++-
cloud/src/meta-service/meta_service_resource.cpp | 4 ++
cloud/test/meta_service_http_test.cpp | 79 ++++++++++++++++++++++++
3 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index 9e9ff38c2ec..8a1a5306194 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -19,7 +19,9 @@
#include <brpc/controller.h>
#include <gen_cpp/cloud.pb.h>
+#include <openssl/md5.h>
+#include <iomanip>
#include <memory>
#include <string>
#include <string_view>
@@ -29,12 +31,26 @@
#include "common/logging.h"
#include "common/stopwatch.h"
#include "common/util.h"
+#include "cpp/sync_point.h"
#include "meta-service/keys.h"
#include "meta-service/txn_kv.h"
#include "meta-service/txn_kv_error.h"
#include "resource-manager/resource_manager.h"
namespace doris::cloud {
+inline std::string md5(const std::string& str) {
+ unsigned char digest[MD5_DIGEST_LENGTH];
+ MD5_CTX context;
+ MD5_Init(&context);
+ MD5_Update(&context, str.c_str(), str.length());
+ MD5_Final(digest, &context);
+
+ std::ostringstream ss;
+ for (unsigned char i : digest) {
+ ss << std::setw(2) << std::setfill('0') << std::hex << (int)i;
+ }
+ return ss.str();
+}
template <class Request>
void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const
Request* req) {
@@ -101,7 +117,34 @@ void finish_rpc(std::string_view func_name,
brpc::Controller* ctrl, Response* re
LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
<< " status=" << res->status().ShortDebugString()
<< " delete_bitmap_size=" <<
res->segment_delete_bitmaps_size();
-
+ } else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
+ std::is_same_v<Response, GetStageResponse>) {
+ std::string debug_string = res->DebugString();
+ // Start position for searching "sk" fields
+ size_t pos = 0;
+ // Iterate through the string and find all occurrences of "sk: "
+ while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) {
+ // Find the start and end of the "sk" value (assumed to be within
quotes)
+ // Start after the quote
+ size_t sk_value_start = debug_string.find('\"', pos) + 1;
+ // End at the next quote
+ size_t sk_value_end = debug_string.find('\"', sk_value_start);
+
+ // Extract the "sk" value
+ std::string sk_value =
+ debug_string.substr(sk_value_start, sk_value_end -
sk_value_start);
+ // Encrypt the "sk" value with MD5
+ std::string encrypted_sk = "md5: " + md5(sk_value);
+
+ // Replace the original "sk" value with the encrypted MD5 value
+ debug_string.replace(sk_value_start, sk_value_end -
sk_value_start, encrypted_sk);
+
+ // Move the position to the end of the current "sk" field and
continue searching
+ pos = sk_value_end;
+ }
+ TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
+ LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
+ << " response=" << debug_string;
} else {
LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
<< " response=" << res->ShortDebugString();
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 92020005c3a..cc459c090bf 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -203,6 +203,8 @@ void
MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
GetObjStoreInfoResponse* response,
::google::protobuf::Closure* done) {
RPC_PREPROCESS(get_obj_store_info);
+ TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response);
+ TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return");
// Prepare data
std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
@@ -2600,6 +2602,8 @@ void
MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
const GetStageRequest* request,
GetStageResponse* response,
::google::protobuf::Closure* done) {
RPC_PREPROCESS(get_stage);
+ TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response);
+ TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return");
std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index e49628fcb3a..d1b8fd66943 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -1456,4 +1456,83 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) {
}
}
+TEST(MetaServiceHttpTest, get_stage_response_sk) {
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+ [&](...) {
sp->disable_processing(); });
+
+ GetStageResponse res;
+ auto* stage = res.add_stage();
+ stage->mutable_obj_info()->set_ak("stage-ak");
+ stage->mutable_obj_info()->set_sk("stage-sk");
+ auto foo = [res](auto args) {
(*(try_any_cast<GetStageResponse**>(args[0])))->CopyFrom(res); };
+ sp->set_call_back("stage_sk_response", foo);
+ sp->set_call_back("stage_sk_response_return",
+ [](auto&& args) { *try_any_cast<bool*>(args.back()) =
true; });
+
+ auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+ auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr,
rate_limiter);
+
+ auto bar = [](auto args) {
+ std::cout << *try_any_cast<std::string*>(args[0]);
+
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("stage-sk") ==
std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: f497d053066fa4b7d3b1f6564597d233") !=
std::string::npos);
+ };
+ sp->set_call_back("sk_finish_rpc", bar);
+
+ GetStageResponse res1;
+ GetStageRequest req1;
+ brpc::Controller cntl;
+ ms->get_stage(&cntl, &req1, &res1, nullptr);
+}
+
+TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) {
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+ [&](...) {
sp->disable_processing(); });
+
+ GetObjStoreInfoResponse res;
+ auto* obj_info = res.add_obj_info();
+ obj_info->set_ak("obj-store-info-ak1");
+ obj_info->set_sk("obj-store-info-sk1");
+ obj_info = res.add_storage_vault()->mutable_obj_info();
+ obj_info->set_ak("obj-store-info-ak2");
+ obj_info->set_sk("obj-store-info-sk2");
+ auto foo = [res](auto args) {
+ (*(try_any_cast<GetObjStoreInfoResponse**>(args[0])))->CopyFrom(res);
+ };
+ sp->set_call_back("obj-store-info_sk_response", foo);
+ sp->set_call_back("obj-store-info_sk_response_return",
+ [](auto&& args) { *try_any_cast<bool*>(args.back()) =
true; });
+
+ auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+ auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr,
rate_limiter);
+
+ auto bar = [](auto args) {
+ std::cout << *try_any_cast<std::string*>(args[0]);
+
+
EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk1") ==
+ std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: 35d5a637fd9d45a28207a888b751efc4") !=
std::string::npos);
+
+
EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk2") ==
+ std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: 01d7473ae201a2ecdf1f7c064eb81a95") !=
std::string::npos);
+ };
+ sp->set_call_back("sk_finish_rpc", bar);
+
+ GetObjStoreInfoResponse res1;
+ GetObjStoreInfoRequest req1;
+ brpc::Controller cntl;
+ ms->get_obj_store_info(&cntl, &req1, &res1, nullptr);
+}
+
} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
