This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 02d03c43bb [INLONG-10822][SDK] Add message manager for DataProxy C++
SDK (#10824)
02d03c43bb is described below
commit 02d03c43bbfeb5915f462bc66081a8fd75170566
Author: doleyzi <[email protected]>
AuthorDate: Tue Aug 20 19:25:51 2024 +0800
[INLONG-10822][SDK] Add message manager for DataProxy C++ SDK (#10824)
---
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 8 ++
.../dataproxy-sdk-cpp/src/config/sdk_conf.h | 1 +
.../dataproxy-sdk-cpp/src/core/sdk_msg.h | 1 +
.../dataproxy-sdk-cpp/src/manager/msg_manager.h | 97 ++++++++++++++++++++++
.../dataproxy-sdk-cpp/src/utils/capi_constant.h | 1 +
5 files changed, 108 insertions(+)
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index edbc075a67..4240a836dc 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -141,6 +141,7 @@ void SdkConfig::defaultInit() {
need_auth_ = constants::kNeedAuth;
max_instance_ = constants::kMaxInstance;
instance_num_ = 1;
+ enable_share_msg_ = constants::kEnableShareMsg;
}
void SdkConfig::InitThreadParam(const rapidjson::Value &doc) {
@@ -229,6 +230,13 @@ void SdkConfig::InitCacheParam(const rapidjson::Value
&doc) {
} else {
max_cache_num_ = constants::kMaxCacheNum;
}
+
+ if (doc.HasMember("enable_share_msg") && doc["enable_share_msg"].IsBool()) {
+ const rapidjson::Value &obj = doc["enable_share_msg"];
+ enable_share_msg_ = obj.GetBool();
+ } else {
+ enable_share_msg_ = constants::kEnableShareMsg;
+ }
}
void SdkConfig::InitZipParam(const rapidjson::Value &doc) {
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
index 3f46b1a0f4..3a649cfb82 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
@@ -53,6 +53,7 @@ private:
uint32_t max_cache_num_;
uint32_t max_instance_;
uint32_t instance_num_;
+ bool enable_share_msg_;
// thread parameters
uint32_t per_groupid_thread_nums_; // Sending thread per groupid
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/sdk_msg.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/sdk_msg.h
index df7e90f4ac..8ae749a352 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/sdk_msg.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/sdk_msg.h
@@ -48,6 +48,7 @@ struct SdkMsg {
data_pack_format_attr_(attr),
inlong_group_id_(inlong_group_id),
inlong_stream_id_(inlong_stream_id){}
+ SdkMsg() {};
};
using SdkMsgPtr = std::shared_ptr<SdkMsg>;
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/msg_manager.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/msg_manager.h
new file mode 100644
index 0000000000..26aa387083
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/msg_manager.h
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <queue>
+#include "../config/sdk_conf.h"
+#include "../utils/send_buffer.h"
+#include "../utils/logger.h"
+#include "../core/sdk_msg.h"
+
+#ifndef INLONG_MSG_MANAGER_H
+#define INLONG_MSG_MANAGER_H
+namespace inlong {
+class MsgManager {
+ private:
+ std::queue<SdkMsgPtr> msg_queue_;
+ mutable std::mutex mutex_;
+ uint32_t queue_limit_;
+ bool enable_share_msg_;
+ MsgManager() {
+ uint32_t data_capacity_ =
std::max(SdkConfig::getInstance()->max_msg_size_,
SdkConfig::getInstance()->pack_size_);
+ uint32_t buffer_num = SdkConfig::getInstance()->recv_buf_size_ /
data_capacity_;
+ queue_limit_ = std::min(SdkConfig::getInstance()->max_cache_num_,
buffer_num);
+ enable_share_msg_ = SdkConfig::getInstance()->enable_share_msg_;
+ LOG_INFO("Data capacity:" << data_capacity_ << ", buffer_num: " <<
buffer_num << ", limit: " << queue_limit_
+ << ", enable share msg: " << enable_share_msg_);
+ for (int i = 0; i < queue_limit_; i++) {
+ std::shared_ptr<SdkMsg> msg_ptr = std::make_shared<SdkMsg>();
+ if (msg_ptr == nullptr) {
+ LOG_INFO("Msg ptr is null");
+ continue;
+ }
+ AddMsg(msg_ptr);
+ }
+ }
+
+ public:
+ static MsgManager *GetInstance() {
+ static MsgManager instance;
+ return &instance;
+ }
+ SdkMsgPtr GetMsg() {
+ if (!enable_share_msg_) {
+ return nullptr;
+ }
+ std::lock_guard<std::mutex> lck(mutex_);
+ if (msg_queue_.empty()) {
+ return nullptr;
+ }
+ SdkMsgPtr buf = msg_queue_.front();
+ msg_queue_.pop();
+ return buf;
+ }
+ void AddMsg(const SdkMsgPtr &msg_ptr) {
+ if (nullptr == msg_ptr || !enable_share_msg_) {
+ return;
+ }
+ std::lock_guard<std::mutex> lck(mutex_);
+ if (msg_queue_.size() > queue_limit_) {
+ return;
+ }
+ msg_queue_.emplace(msg_ptr);
+ }
+
+ void AddMsg(const std::vector<SdkMsgPtr> &user_msg_vector) {
+ if (!enable_share_msg_) {
+ return;
+ }
+ std::lock_guard<std::mutex> lck(mutex_);
+ for (auto it : user_msg_vector) {
+ if (nullptr == it) {
+ continue;
+ }
+ if (msg_queue_.size() > queue_limit_) {
+ return;
+ }
+ msg_queue_.emplace(std::move(it));
+ }
+ }
+};
+} // namespace inlong
+#endif // INLONG_MSG_MANAGER_H
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
index dbac24f6a6..616f018b90 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
@@ -50,6 +50,7 @@ static const int32_t kLoadBalanceInterval = 300000;
static const int32_t kHeartBeatInterval = 60000;
static const bool kEnableBalance = true;
static const bool kEnableLocalCache = true;
+static const bool kEnableShareMsg = true;
static const bool kEnablePack = true;
static const uint32_t kPackSize = 409600;
