This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
The following commit(s) were added to refs/heads/main by this push:
new 325f852 Support building with MSVC (#23)
325f852 is described below
commit 325f852df87dc15d68dd87fd540cf71e61d93281
Author: Yunze Xu <[email protected]>
AuthorDate: Wed Nov 2 11:41:40 2022 +0800
Support building with MSVC (#23)
Fixes https://github.com/apache/pulsar-client-python/issues/7
The Python client cannot be built with MSVC.
CMakeLists.txt:
1. Boost.Python cannot be found on Windows. The component of Boost
cannot be `python3`. It should be a specific version like
`python310`. Therefore, find all possible components from 3.10 to 3.7
until the first component is available.
2. For MSVC, link to `pulsarWithDeps.lib` when `LINK_STATIC` is `ON` and
set the `MSVC_RUNTIME_LIBRARY` target property to `MultiThreaded`.
3. Change the suffix from `.so` to `.pyd` because Python on Windows
recognizes `*.pyd` as the dynamic library.
4. Link to Python library with MSVC, otherwise the symbos cannot be
found when linking `boost-python`.
README: tell users how to build Python client on Windows.
Add a workflow to build Python wheels of versions 3.7 to 3.10 on Windows
and verify the build will succeed.
With this PR, Windows users still need to put the related
`boost_python*.dll` under the system path. In future, the `boost-python`
dependency will be removed. See
https://github.com/apache/pulsar-client-python/issues/24.
---
.github/workflows/ci-pr-validation.yaml | 85 ++++++++++++++++++++++++++++++++-
.gitignore | 5 +-
.gitmodules | 3 ++
CMakeLists.txt | 62 +++++++++++++++++++-----
README.md | 32 ++++++++++++-
setup.py | 9 +++-
vcpkg | 1 +
vcpkg-3.10.json | 12 +++++
vcpkg-3.7.json | 18 +++++++
vcpkg-3.8.json | 18 +++++++
vcpkg-3.9.json | 12 +++++
vcpkg.json | 6 +++
12 files changed, 247 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/ci-pr-validation.yaml
b/.github/workflows/ci-pr-validation.yaml
index e1b7434..6f3f894 100644
--- a/.github/workflows/ci-pr-validation.yaml
+++ b/.github/workflows/ci-pr-validation.yaml
@@ -67,6 +67,7 @@ jobs:
linux-wheel:
name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} -
${{matrix.cpu.platform}}
+ needs: unit-tests
runs-on: ubuntu-22.04
timeout-minutes: 300
@@ -118,6 +119,7 @@ jobs:
mac-wheels:
name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
+ needs: unit-tests
runs-on: macos-12
timeout-minutes: 300
@@ -155,11 +157,92 @@ jobs:
- name: Build and test Mac wheels
run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
+ windows-wheels:
+ name: "Python ${{ matrix.python.version }} Wheel on ${{
matrix.windows.name }}"
+ needs: unit-tests
+ runs-on: ${{ matrix.windows.os }}
+ timeout-minutes: 120
+
+ env:
+ PULSAR_CPP_DIR: 'C:\\pulsar-cpp'
+ strategy:
+ fail-fast: false
+ matrix:
+ windows:
+ - name: 'Windows x64'
+ os: windows-2022
+ arch: '-A x64'
+ triplet: 'x64-windows'
+ python:
+ - version: '3.7'
+ - version: '3.8'
+ - version: '3.9'
+ - version: '3.10'
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python.version }}
+
+ - name: Prepare vcpkg.json
+ shell: bash
+ run: |
+ python --version
+ cp -f vcpkg-${{ matrix.python.version }}.json vcpkg.json
+ cat vcpkg.json
+
+ - name: Download Pulsar C++ client on Windows
+ shell: bash
+ run: |
+ mkdir -p ${{ env.PULSAR_CPP_DIR }}
+ cd ${{ env.PULSAR_CPP_DIR }}
+ # TODO: switch to official releases
+ curl -O -L
https://github.com/BewareMyPower/pulsar-client-cpp/releases/download/v3.1.0-rc-20221028/${{
matrix.windows.triplet }}-static.zip
+ unzip -q ${{ matrix.windows.triplet }}-static.zip
+ ls -l ${{ env.PULSAR_CPP_DIR }}
+
+ - name: Cache Vcpkg
+ uses: actions/cache@v3
+ id: cache-vcpkg
+ with:
+ path: build/vcpkg_installed
+ key: ${{ matrix.python.version }}-${{
hashFiles(format('vcpkg-{0}.json', matrix.python.version)) }}
+
+ - name: Install dependencies and configure CMake
+ shell: bash
+ run: |
+ COMMIT_ID=$(grep baseline vcpkg.json | sed 's/[",]//g' | awk '{print
$2}')
+ cd vcpkg
+ echo "git fetch origin $COMMIT_ID"
+ git fetch origin $COMMIT_ID
+ cd -
+ cmake -B build ${{ matrix.windows.arch }} \
+ -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
+ -DUSE_VCPKG=ON \
+ -DLINK_STATIC=ON
+
+ - name: Build Python wheel
+ shell: bash
+ run: |
+ cmake --build build --config Release --target install
+ python -m pip install wheel
+ python setup.py bdist_wheel
+ python -m pip install ./dist/*.whl
+ cp ./build/Release/boost_python*.dll .
+ echo "The extra DLLs:"
+ ls -l *.dll
+ python -c 'import pulsar; c =
pulsar.Client("pulsar://localhost:6650"); c.close()'
+
+
# Job that will be required to complete and depends on all the other jobs
check-completion:
name: Check Completion
runs-on: ubuntu-latest
- needs: [unit-tests, linux-wheel, mac-wheels]
+ needs: [unit-tests, linux-wheel, mac-wheels, windows-wheels]
steps:
- run: true
diff --git a/.gitignore b/.gitignore
index ee9119f..72f931b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,4 +13,7 @@ __pycache__
.pulsar-mac-wheels-cache
.DS_Store
wheelhouse
-.pulsar-mac-build
\ No newline at end of file
+.pulsar-mac-build
+vcpkg_installed/
+*.pyd
+*.lib
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..a0a57f3
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "vcpkg"]
+ path = vcpkg
+ url = https://github.com/microsoft/vcpkg.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 10a70ec..a60e1c2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,10 +17,15 @@
# under the License.
#
-project (pulsar-client-python)
cmake_minimum_required(VERSION 3.18)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
+option(USE_VCPKG "Use Vcpkg to install dependencies" OFF)
+if (USE_VCPKG)
+ set(CMAKE_TOOLCHAIN_FILE
"${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake"
+ CACHE STRING "Vcpkg toolchain file")
+endif ()
+project (pulsar-client-python)
option(LINK_STATIC "Link against static libraries" OFF)
MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC})
@@ -29,10 +34,19 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
find_package(Threads REQUIRED)
MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
+if (MSVC)
+ add_compile_options(/wd4819)
+endif ()
+
if (LINK_STATIC)
- find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+ if (MSVC)
+ find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib)
+ else ()
+ find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+ endif ()
+ add_definitions("-DPULSAR_STATIC")
else()
- find_library(PULSAR_LIBRARY NAMES libpulsar.so libpulsar.dylib)
+ find_library(PULSAR_LIBRARY NAMES pulsar libpulsar)
endif()
message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
@@ -44,10 +58,23 @@ SET(CMAKE_CXX_STANDARD 11)
find_package (Python3 REQUIRED COMPONENTS Development.Module)
MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})
+find_package(Boost REQUIRED ${Boost_INCLUDE_DIRS})
+message(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
+
SET(Boost_USE_STATIC_LIBS ${LINK_STATIC})
-find_package(Boost REQUIRED COMPONENTS python3)
-MESSAGE(STATUS "Boost Python3: " ${Boost_PYTHON3_LIBRARY})
-MESSAGE(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
+
+set(BOOST_PYTHON_NAME_LIST python3 python310 python39 python38 python37)
+foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
+ find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
+ if (${Boost_FOUND})
+ set(BOOST_PYTHON_COMPONENT_FOUND ${BOOST_PYTHON_NAME})
+ message(STATUS "Found Boost COMPONENTS "
${BOOST_PYTHON_COMPONENT_FOUND})
+ break ()
+ endif ()
+endforeach ()
+if (NOT BOOST_PYTHON_COMPONENT_FOUND)
+ message(FATAL_ERROR "Could not find Boost Python library")
+endif ()
########################################################################################################################
@@ -68,8 +95,11 @@ ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
src/utils.cc
)
-SET(CMAKE_SHARED_LIBRARY_PREFIX )
-SET(CMAKE_SHARED_LIBRARY_SUFFIX .so)
+if (MSVC)
+ set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd)
+else ()
+ set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
+endif ()
if (NOT APPLE AND NOT MSVC)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_PYTHON}")
@@ -80,12 +110,17 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
endif()
# Try all possible boost-python variable namings
-set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
- ${Boost_PYTHON3_LIBRARY})
+set(PYTHON_WRAPPER_LIBS
+ ${PULSAR_LIBRARY}
+ Boost::${BOOST_PYTHON_COMPONENT_FOUND}
+)
+if (MSVC)
+ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module)
+endif ()
message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
-if (LINK_STATIC)
+if (LINK_STATIC AND NOT MSVC)
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
# We need to include all the static libs individually because we cannot
easily create a universal2 libpulsar.a
@@ -125,9 +160,14 @@ if (LINK_STATIC)
endif()
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
endif ()
+elseif (LINK_STATIC) # MSVC
+ set_property(TARGET _pulsar PROPERTY
+ MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+ target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
else()
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
endif ()
+install(TARGETS _pulsar DESTINATION ${CMAKE_SOURCE_DIR})
find_package(ClangTools)
set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
diff --git a/README.md b/README.md
index 0d403cd..b0857a0 100644
--- a/README.md
+++ b/README.md
@@ -31,13 +31,39 @@
## Install the Python wheel
+### Windows (with Vcpkg)
+
+First, install the dependencies via
[Vcpkg](https://github.com/microsoft/vcpkg).
+
+```PowerShell
+vcpkg install --feature-flags=manifests --triplet x64-windows
+```
+
+> NOTE: For Windows 32-bit library, change `x64-windows` to `x86-windows`, see
[here](https://github.com/microsoft/vcpkg/tree/master/triplets) for all
available triplets.
+
+Then, build and install the Python wheel.
+
+```PowerShell
+# Assuming the Pulsar C++ client has been installed under the `PULSAR_CPP`
directory.
+cmake -B build -DUSE_VCPKG=ON -DCMAKE_PREFIX_PATH="$env:PULSAR_CPP"
-DLINK_STATIC=ON
+cmake --build build --config Release
+cmake --install build
+py setup.py bdist_wheel
+py -m pip install ./dist/pulsar_client-*.whl
+```
+
+Since the Python client links to Boost.Python dynamically, you have to copy
the dll (e.g. `boost_python310-vc142-mt-x64-1_80.dll`) into the system path
(the `PATH` environment variable). If the `-DLINK_STATIC=ON` option is not
specified, you have to copy the `pulsar.dll` into the system path as well.
+
+### Linux or macOS
+
+Assuming the Pulsar C++ client and Boost.Python have been installed under the
system path.
+
```bash
cmake -B build
cmake --build build -j8
-cp build/_pulsar.so .
+cmake --install build
./setup.py bdist_wheel
pip3 install dist/pulsar_client-*.whl --force-reinstall
-rm _pulsar.so
```
> **NOTE**
@@ -45,6 +71,8 @@ rm _pulsar.so
> 1. Here a separate `build` directory is created to store all CMake temporary
> files. However, the `setup.py` requires the `_pulsar.so` is under the
> project directory.
> 2. Add the `--force-reinstall` option to overwrite the existing Python wheel
> in case your system has already installed a wheel before.
+## Running examples
+
You can run `python3 -c 'import pulsar'` to see whether the wheel has been
installed successfully. If it failed, check whether dependencies (e.g.
`libpulsar.so`) are in the system path. If not, make sure the dependencies are
in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS).
Then you can run examples as a simple end-to-end test.
diff --git a/setup.py b/setup.py
index f20446f..046b2bb 100755
--- a/setup.py
+++ b/setup.py
@@ -21,6 +21,7 @@
from setuptools import setup
from distutils.core import Extension
from os import environ, path
+import platform
from distutils.command import build_ext
@@ -58,7 +59,13 @@ class my_build_ext(build_ext.build_ext):
except OSError as e:
if e.errno != 17: # already exists
raise
- shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name))
+ if 'Windows' in platform.platform():
+ shutil.copyfile('_pulsar.pyd', self.get_ext_fullpath(ext.name))
+ else:
+ try:
+ shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name))
+ except FileNotFoundError:
+ shutil.copyfile('lib_pulsar.so',
self.get_ext_fullpath(ext.name))
# Core Client dependencies
diff --git a/vcpkg b/vcpkg
new file mode 160000
index 0000000..2537044
--- /dev/null
+++ b/vcpkg
@@ -0,0 +1 @@
+Subproject commit 253704407ae68efa37bf8f5b59b3e06dd40d3d3f
diff --git a/vcpkg-3.10.json b/vcpkg-3.10.json
new file mode 100644
index 0000000..fdb1128
--- /dev/null
+++ b/vcpkg-3.10.json
@@ -0,0 +1,12 @@
+{
+ "name": "pulsar-python",
+ "version": "3.0.0",
+ "description": "Pulsar Python SDK (Python 3.10)",
+ "dependencies": [
+ {
+ "name": "boost-python",
+ "version>=": "1.79.0"
+ }
+ ],
+ "builtin-baseline": "c266859544a3cdcfd952d218039c55a268863740"
+}
diff --git a/vcpkg-3.7.json b/vcpkg-3.7.json
new file mode 100644
index 0000000..1b7b846
--- /dev/null
+++ b/vcpkg-3.7.json
@@ -0,0 +1,18 @@
+{
+ "name": "pulsar-python",
+ "version": "3.0.0",
+ "description": "Pulsar Python SDK (Python 3.7)",
+ "dependencies": [
+ {
+ "name": "boost-python",
+ "version>=": "1.76.0"
+ }
+ ],
+ "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379",
+ "overrides": [
+ {
+ "name": "python3",
+ "version": "3.7.3"
+ }
+ ]
+}
diff --git a/vcpkg-3.8.json b/vcpkg-3.8.json
new file mode 100644
index 0000000..b210e87
--- /dev/null
+++ b/vcpkg-3.8.json
@@ -0,0 +1,18 @@
+{
+ "name": "pulsar-python",
+ "version": "3.0.0",
+ "description": "Pulsar Python SDK (Python 3.8)",
+ "dependencies": [
+ {
+ "name": "boost-python",
+ "version>=": "1.76.0"
+ }
+ ],
+ "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379",
+ "overrides": [
+ {
+ "name": "python3",
+ "version": "3.8.3"
+ }
+ ]
+}
diff --git a/vcpkg-3.9.json b/vcpkg-3.9.json
new file mode 100644
index 0000000..250a1ee
--- /dev/null
+++ b/vcpkg-3.9.json
@@ -0,0 +1,12 @@
+{
+ "name": "pulsar-python",
+ "version": "3.0.0",
+ "description": "Pulsar Python SDK (Python 3.9)",
+ "dependencies": [
+ {
+ "name": "boost-python",
+ "version>=": "1.76.0"
+ }
+ ],
+ "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379"
+}
diff --git a/vcpkg.json b/vcpkg.json
new file mode 100644
index 0000000..ef5c7c2
--- /dev/null
+++ b/vcpkg.json
@@ -0,0 +1,6 @@
+{
+ "name": "pulsar-python",
+ "version": "3.0.0",
+ "description": "Pulsar Python SDK",
+ "dependencies": ["boost-python"]
+}
