This is an automated email from the ASF dual-hosted git repository.
xyz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
The following commit(s) were added to refs/heads/main by this push:
new 1554e39 Fixed building of Linux wheel files (#45)
1554e39 is described below
commit 1554e39e105fe80a6427801284a6b11fdc23bff2
Author: Matteo Merli <[email protected]>
AuthorDate: Fri Nov 11 21:00:26 2022 -0800
Fixed building of Linux wheel files (#45)
The current build process for Python wheels is to fetch the RPM packages of
the C++ release and use the `libpulsar.so` from there.
There are some issues due to incompatibilities between different versions
of `libstdc++`, causing segmentation faults. In particular problems with
callbacks and `std::function` objects compiled in different environments.
To avoid these issues, we need to compile the C++ client lib when we're
building the Python wrapper.
---
build-support/copy-deps-versionfile.sh | 1 +
dependencies.yaml | 2 -
pkg/build-wheel-inside-docker.sh | 2 -
pkg/manylinux2014/Dockerfile | 70 +++++++++++++++++++++++++++++++++-
pkg/manylinux_musl/Dockerfile | 68 +++++++++++++++++++++++++++++++++
5 files changed, 138 insertions(+), 5 deletions(-)
diff --git a/build-support/copy-deps-versionfile.sh
b/build-support/copy-deps-versionfile.sh
index 0a36d10..3e42105 100755
--- a/build-support/copy-deps-versionfile.sh
+++ b/build-support/copy-deps-versionfile.sh
@@ -25,5 +25,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
for dir in manylinux2014 manylinux_musl; do
mkdir -p pkg/$dir/.build
cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
+ cp $ROOT_DIR/pulsar-client-cpp-version.txt pkg/$dir/.build
cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
done
diff --git a/dependencies.yaml b/dependencies.yaml
index 262b100..3d76f96 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -18,8 +18,6 @@
#
boost: 1.80.0
-
-# Only used for MacOS builds
cmake: 3.24.2
protobuf: 3.20.0
zlib: 1.2.13
diff --git a/pkg/build-wheel-inside-docker.sh b/pkg/build-wheel-inside-docker.sh
index a477b35..09b4304 100755
--- a/pkg/build-wheel-inside-docker.sh
+++ b/pkg/build-wheel-inside-docker.sh
@@ -22,8 +22,6 @@ set -e -x
cd /pulsar-client-python
-build-support/install-cpp-client.sh
-
rm -f CMakeCache.txt CMakeFiles
cmake . \
diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile
index 7355585..70f5e31 100644
--- a/pkg/manylinux2014/Dockerfile
+++ b/pkg/manylinux2014/Dockerfile
@@ -22,7 +22,6 @@ FROM quay.io/pypa/manylinux2014_${ARCH}
ARG PYTHON_VERSION
ARG PYTHON_SPEC
-ARG PLATFORM
ARG ARCH
ENV PYTHON_VERSION=${PYTHON_VERSION}
@@ -36,6 +35,7 @@ ENV PYTHON_LIBRARIES
/opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
RUN pip3 install pyyaml
ADD .build/dependencies.yaml /
+ADD .build/pulsar-client-cpp-version.txt /
ADD .build/dep-version.py /usr/local/bin
# Download and compile boost
@@ -54,3 +54,71 @@ RUN CMAKE_VERSION=$(dep-version.py cmake) && \
cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \
cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \
rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH}
cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz
+
+# Download and compile protobuf
+RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
+ curl -O -L
https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
&& \
+ tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
+ cd protobuf-${PROTOBUF_VERSION}/ && \
+ CXXFLAGS=-fPIC ./configure && \
+ make -j8 && make install && ldconfig && \
+ rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
/protobuf-${PROTOBUF_VERSION}
+
+# ZLib
+RUN ZLIB_VERSION=$(dep-version.py zlib) && \
+ curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz
&& \
+ tar xfz v${ZLIB_VERSION}.tar.gz && \
+ cd zlib-${ZLIB_VERSION} && \
+ CFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
+
+# Zstandard
+RUN ZSTD_VERSION=$(dep-version.py zstd) && \
+ curl -O -L
https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
&& \
+ tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
+ cd zstd-${ZSTD_VERSION} && \
+ CFLAGS="-fPIC -O3" make -j8 && \
+ make install && \
+ rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
+
+# Snappy
+RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
+ curl -O -L
https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
+ tar xfz ${SNAPPY_VERSION}.tar.gz && \
+ cd snappy-${SNAPPY_VERSION} && \
+ CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF
-DSNAPPY_BUILD_BENCHMARKS=OFF && \
+ make -j8 && make install && \
+ rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
+
+RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
+ OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
+ curl -O -L
https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
&& \
+ tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
+ cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
+ ./config -fPIC --prefix=/usr/local/ssl/ && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
/openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
+
+ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib
+ENV OPENSSL_ROOT_DIR /usr/local/ssl/
+
+# LibCurl
+RUN CURL_VERSION=$(dep-version.py curl) && \
+ CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
+ curl -O -L
https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz
&& \
+ tar xfz curl-${CURL_VERSION}.tar.gz && \
+ cd curl-${CURL_VERSION} && \
+ CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
+ make -j8 && make install && \
+ rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
+
+# Pulsar client C++
+RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \
+ curl -O -L
https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
&& \
+ tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
+ cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \
+ cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF
-DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \
+ make -j8 && \
+ make install && \
+ rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}
apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile
index f402bec..796d355 100644
--- a/pkg/manylinux_musl/Dockerfile
+++ b/pkg/manylinux_musl/Dockerfile
@@ -38,6 +38,7 @@ RUN pip install pyyaml
RUN apk add cmake
ADD .build/dependencies.yaml /
+ADD .build/pulsar-client-cpp-version.txt /
ADD .build/dep-version.py /usr/local/bin
# Download and compile boost
@@ -51,3 +52,70 @@ RUN BOOST_VERSION=$(dep-version.py boost) && \
rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz
/boost_${BOOST_VERSION_UNDESRSCORE}
+# Download and compile protobuf
+RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
+ curl -O -L
https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
&& \
+ tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
+ cd protobuf-${PROTOBUF_VERSION}/ && \
+ CXXFLAGS=-fPIC ./configure && \
+ make -j8 && make install && \
+ rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
/protobuf-${PROTOBUF_VERSION}
+
+# ZLib
+RUN ZLIB_VERSION=$(dep-version.py zlib) && \
+ curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz
&& \
+ tar xfz v${ZLIB_VERSION}.tar.gz && \
+ cd zlib-${ZLIB_VERSION} && \
+ CFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
+
+# Zstandard
+RUN ZSTD_VERSION=$(dep-version.py zstd) && \
+ curl -O -L
https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
&& \
+ tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
+ cd zstd-${ZSTD_VERSION} && \
+ CFLAGS="-fPIC -O3" make -j8 && \
+ make install && \
+ rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
+
+# Snappy
+RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
+ curl -O -L
https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
+ tar xfz ${SNAPPY_VERSION}.tar.gz && \
+ cd snappy-${SNAPPY_VERSION} && \
+ CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF
-DSNAPPY_BUILD_BENCHMARKS=OFF && \
+ make -j8 && make install && \
+ rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
+
+RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
+ OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
+ curl -O -L
https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
&& \
+ tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
+ cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
+ ./config -fPIC --prefix=/usr/local/ssl/ && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
/openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
+
+ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib
+ENV OPENSSL_ROOT_DIR /usr/local/ssl/
+
+# LibCurl
+RUN CURL_VERSION=$(dep-version.py curl) && \
+ CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
+ curl -O -L
https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz
&& \
+ tar xfz curl-${CURL_VERSION}.tar.gz && \
+ cd curl-${CURL_VERSION} && \
+ CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
+ make -j8 && make install && \
+ rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
+
+# Pulsar client C++
+RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \
+ curl -O -L
https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
&& \
+ tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
+ cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \
+ cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF
-DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \
+ make -j8 && \
+ make install && \
+ rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}
apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
