This is an automated email from the ASF dual-hosted git repository.
luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new d5189626ab [INLONG-10531][SDK] Add InLong dataproxy python SDK based
on C++ SDK (#10538)
d5189626ab is described below
commit d5189626aba1a03448d55de84da10d69095dc7a1
Author: LeeWY <[email protected]>
AuthorDate: Sat Jun 29 18:24:35 2024 +0800
[INLONG-10531][SDK] Add InLong dataproxy python SDK based on C++ SDK
(#10538)
* [INLONG-10531][SDK] Add InLong dataproxy python SDK based on C++ SDK
* [INLONG-10531][SDK] Improve the callback function part
* [INLONG-10531][SDK] Optimize the build script
---------
Co-authored-by: jameswyli <[email protected]>
---
.../dataproxy-sdk-python/CMakeLists.txt | 38 +++++++++++
.../dataproxy-sdk-python/README.md | 41 ++++++++++++
.../dataproxy-sdk-python/build.sh | 73 ++++++++++++++++++++++
.../dataproxy-sdk-python/inlong_dataproxy.cpp | 61 ++++++++++++++++++
4 files changed, 213 insertions(+)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
new file mode 100644
index 0000000000..80791b1c02
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cmake_minimum_required(VERSION 3.5)
+project(dataproxy-sdk-python)
+
+set(CMAKE_CXX_STANDARD 11)
+
+include_directories("./dataproxy-sdk-cpp/src/core")
+
+include_directories("./dataproxy-sdk-cpp/third_party/lib")
+include_directories("./dataproxy-sdk-cpp/third_party/lib64")
+
+add_subdirectory(pybind11)
+add_subdirectory(dataproxy-sdk-cpp)
+
+link_directories("./dataproxy-sdk-cpp/third_party/lib")
+link_directories("./dataproxy-sdk-cpp/third_party/lib64")
+
+pybind11_add_module(inlong_dataproxy inlong_dataproxy.cpp)
+
+target_link_libraries(inlong_dataproxy PRIVATE pybind11::module dataproxy_sdk)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md
new file mode 100644
index 0000000000..d85a8632f3
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md
@@ -0,0 +1,41 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# DataProxy-SDK-Python
+Dataproxy-SDK Python version, used for sending data to InLong dataproxy.
+
+InLong Dataproxy Python SDK is a wrapper over the existing [C++
SDK](https://github.com/apache/inlong/tree/master/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp)
and exposes all of the same features.
+
+## Prerequisites
+- CMake 3.5+
+- Python 3.6+
+
+## Build
+Go to the dataproxy-sdk-python root directory, and run
+
+```bash
+chmod +x ./build.sh
+./build.sh
+```
+
+After the build process finished, you can import the package (`import
inlong_dataproxy`) in your python project to use InLong dataproxy.
+
+> **Note**: When the C++ SDK or the version of Python you're using is updated,
you'll need to rebuild it by re-executing the `build.sh` script
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
new file mode 100755
index 0000000000..388edcbc61
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Initialize the configuration files of inlong components
+#
+
+#!/bin/bash
+
+BASE_DIR=$(pwd)
+
+# Check CMake version
+CMAKE_VERSION=$(cmake --version | head -n 1 | cut -d " " -f 3)
+CMAKE_REQUIRED="3.5"
+if [ "$(printf '%s\n' "$CMAKE_REQUIRED" "$CMAKE_VERSION" | sort -V | head
-n1)" != "$CMAKE_REQUIRED" ]; then
+ echo "CMake version must be greater than or equal to $CMAKE_REQUIRED"
+ exit 1
+fi
+
+# Check Python version
+PYTHON_VERSION=$(python --version 2>&1 | cut -d " " -f 2)
+PYTHON_REQUIRED="3.6"
+if [ "$(printf '%s\n' "$PYTHON_REQUIRED" "$PYTHON_VERSION" | sort -V | head
-n1)" != "$PYTHON_REQUIRED" ]; then
+ echo "Python version must be greater than or equal to $PYTHON_REQUIRED"
+ exit 1
+fi
+
+# Clone and build pybind11
+git clone https://github.com/pybind/pybind11.git
+cd pybind11
+mkdir build && cd build
+cmake ..
+cmake --build . --config Release --target check
+make check -j 4
+cd $BASE_DIR
+
+# Clone and build dataproxy-sdk-cpp
+git clone https://github.com/apache/inlong.git
+mv ./inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp ./
+rm -r ./inlong
+cd ./dataproxy-sdk-cpp
+chmod +x ./build.sh
+./build.sh
+cd $BASE_DIR
+
+# Build Python SDK
+if [ -d "./build" ]; then
+ rm -r ./build
+fi
+mkdir build && cd build
+cmake ..
+make
+cd $BASE_DIR
+
+# Get Python site-packages directory
+SITE_PACKAGES_DIR=$(python -c "import site; print(site.getsitepackages()[0])")
+
+# Copy generated .so file to site-packages directory
+find ./build -name "*.so" -print0 | xargs -0 -I {} bash -c 'rm -f $0/$1; cp $1
$0' $SITE_PACKAGES_DIR {}
+
+# Clean
+rm -r ./pybind11 ./dataproxy-sdk-cpp
\ No newline at end of file
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
new file mode 100644
index 0000000000..26c260f3f9
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <inlong_api.h>
+
+namespace py = pybind11;
+using namespace inlong;
+
+class PyInLongApi : public InLongApi {
+public:
+ int32_t PySend(const char *inlong_group_id, const char *inlong_stream_id,
const char *msg, int32_t msg_len, py::function callback_func) {
+ py_callback = callback_func;
+ return Send(inlong_group_id, inlong_stream_id, msg, msg_len,
&PyInLongApi::CallbackFunc);
+ }
+
+private:
+ static py::function py_callback;
+
+ static int CallbackFunc(const char *a, const char *b, const char *c,
int32_t d, const int64_t e, const char *f) {
+ if (py_callback) {
+ try {
+ return py_callback(a, b, c, d, e, f).cast<int>();
+ } catch (const py::error_already_set &e) {
+ // Handle Python exception
+ return -1;
+ }
+ }
+ return 0;
+ }
+};
+
+py::function PyInLongApi::py_callback;
+
+PYBIND11_MODULE(inlong_dataproxy, m) {
+ m.doc() = "This module provides InLong dataproxy api to send message to
InLong dataproxy.";
+
+ py::class_<PyInLongApi>(m, "InLongApi")
+ .def(py::init<>())
+ .def("init_api", &PyInLongApi::InitApi, py::arg("config_path"))
+ .def("add_bid", &PyInLongApi::AddBid, py::arg("group_ids"))
+ .def("send", &PyInLongApi::PySend, py::arg("inlong_group_id"),
py::arg("inlong_stream_id"), py::arg("msg"), py::arg("msg_len"),
py::arg("callback_func") = nullptr)
+ .def("close_api", &PyInLongApi::CloseApi, py::arg("max_waitms"));
+}
