This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 1a0c8dc Added docker files and script to build static client library and Python bindings on Linux (#1793) 1a0c8dc is described below commit 1a0c8dcea29feb3d2cee71e4fcc3cab6b06e65bf Author: Matteo Merli <mme...@apache.org> AuthorDate: Thu May 17 16:29:53 2018 -0700 Added docker files and script to build static client library and Python bindings on Linux (#1793) * Added docker files and script to build static client library and Python bindings on Linux * Added license headers --- pulsar-client-cpp/docker/Dockerfile | 144 +++++++++++++++++++++ .../docker/build-client-lib-within-docker.sh | 34 +++++ pulsar-client-cpp/docker/build-client-lib.sh | 44 +++++++ .../docker/build-wheel-file-within-docker.sh | 35 +++++ pulsar-client-cpp/docker/build-wheels.sh | 48 +++++++ pulsar-client-cpp/docker/create-images.sh | 48 +++++++ pulsar-client-cpp/docker/push-images.sh | 51 ++++++++ 7 files changed, 404 insertions(+) diff --git a/pulsar-client-cpp/docker/Dockerfile b/pulsar-client-cpp/docker/Dockerfile new file mode 100644 index 0000000..1df9545 --- /dev/null +++ b/pulsar-client-cpp/docker/Dockerfile @@ -0,0 +1,144 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +FROM quay.io/pypa/manylinux1_x86_64 + +RUN yum install -y gtest-devel + +ARG PYTHON_VERSION +ARG PYTHON_SPEC + +ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV PYTHON_SPEC=${PYTHON_SPEC} + +ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" + +RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION} + +#################################### +# These dependencies can be found in Ubuntu but they're not compiled with -fPIC, +# so they cannot be statically linked into a shared library +#################################### + +# ZLib +RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \ + tar xvfz zlib-1.2.11.tar.gz && \ + cd zlib-1.2.11 && \ + CFLAGS="-fPIC -O3" ./configure && \ + make && make install && \ + rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11 + +# Compile OpenSSL +RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_0_2l.tar.gz && \ + tar xvfz OpenSSL_1_0_2l.tar.gz && \ + cd openssl-OpenSSL_1_0_2l/ && \ + ./Configure -fPIC no-shared linux-x86_64 && \ + make && make install && \ + rm -rf /OpenSSL_1_0_2l.tar.gz /openssl-OpenSSL_1_0_2l + +# Download and compile boost +RUN curl -O -L https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz && \ + tar xvfz boost_1_64_0.tar.gz && \ + cd /boost_1_64_0 && \ + ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \ + ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ + rm -rf /boost_1_64_0.tar.gz /boost_1_64_0 + +# Download and copile protoubf +RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \ + tar xvfz protobuf-cpp-3.3.0.tar.gz && \ + cd protobuf-3.3.0/ && \ + CXXFLAGS=-fPIC ./configure && \ + make && make install && ldconfig && \ + rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0 + +# Compile APR +RUN curl -O -L http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \ + tar xvfz apr-1.5.2.tar.gz && \ + cd apr-1.5.2 && \ + CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \ + make && make install && \ + rm -rf /apr-1.5.2.tar.gz /apr-1.5.2 + +# Compile APR-Util +RUN curl -O -L http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \ + tar xvfz apr-util-1.5.4.tar.gz && \ + cd apr-util-1.5.4 && \ + CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \ + make && make install && \ + rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4 + +# Libtool +RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \ + tar xvfz libtool-2.4.6.tar.gz && \ + cd libtool-2.4.6 && \ + ./configure && \ + make && make install && \ + rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6 + +# Compile log4cxx +RUN git clone https://github.com/apache/logging-log4cxx.git && \ + cd logging-log4cxx && \ + ./autogen.sh && \ + CXXFLAGS=-fPIC ./configure && \ + make && make install && \ + rm -rf /logging-log4cxx + +# Compile expat +RUN curl -O -L https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \ + tar xfvz R_2_2_0.tar.gz && \ + cd libexpat-R_2_2_0/expat && \ + ./buildconf.sh && \ + CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \ + make && make installlib && \ + rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0 + +RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.8.2.tar.gz && \ + tar xvfz v3.8.2.tar.gz && \ + cd CMake-3.8.2 && \ + ./configure && \ + make && make install && \ + rm -rf /v3.8.2.tar.gz /CMake-3.8.2 + +# Compile JSON CPP +RUN curl -O -L https://github.com/open-source-parsers/jsoncpp/archive/1.8.0.tar.gz && \ + tar xvfz 1.8.0.tar.gz && \ + cd jsoncpp-1.8.0 && \ + cmake . -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \ + make && make install && \ + rm -rf /1.8.0.tar.gz /jsoncpp-1.8.0 + +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_54_0/curl-7.54.0.tar.gz && \ + tar xvfz curl-7.54.0.tar.gz && \ + cd curl-7.54.0 && \ + CFLAGS=-fPIC ./configure && \ + make && make install && \ + rm -rf /curl-7.54.0.tar.gz /curl-7.54.0 + +RUN pip install twine + + +ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include +ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ + +COPY build-wheel-file-within-docker.sh / +COPY build-client-lib-within-docker.sh / diff --git a/pulsar-client-cpp/docker/build-client-lib-within-docker.sh b/pulsar-client-cpp/docker/build-client-lib-within-docker.sh new file mode 100755 index 0000000..3a356cc --- /dev/null +++ b/pulsar-client-cpp/docker/build-client-lib-within-docker.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +set -e -x + +cd /pulsar/pulsar-client-cpp + +find . -name CMakeCache.txt | xargs rm +find . -name CMakeFiles | xargs rm -rf +rm lib/*.pb.* + +cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON \ + -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERSION \ + -DPYTHON_LIBRARY=/opt/python/$PYTHON_SPEC/lib \ + +make pulsarShared pulsarStatic -j4 diff --git a/pulsar-client-cpp/docker/build-client-lib.sh b/pulsar-client-cpp/docker/build-client-lib.sh new file mode 100755 index 0000000..61f2d66 --- /dev/null +++ b/pulsar-client-cpp/docker/build-client-lib.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +set -e + +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR + +PYTHON_VERSIONS=( + '3.6 cp36-cp36m' +) + +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_SPEC=${PY[1]} + echo "--------- Build Client library" + + IMAGE_NAME=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC + + echo "Using image: $IMAGE_NAME" + + docker run -i -v $PWD:/pulsar $IMAGE_NAME /build-client-lib-within-docker.sh +done diff --git a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh new file mode 100755 index 0000000..aa224fe --- /dev/null +++ b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +cd /pulsar/pulsar-client-cpp + +find . -name CMakeCache.txt | xargs rm +find . -name CMakeFiles | xargs rm -rf + +cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERSION \ + -DPYTHON_LIBRARY=/opt/python/$PYTHON_SPEC/lib \ + -DLINK_STATIC=ON + +make clean +make _pulsar + +cd python +python setup.py bdist_wheel diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh new file mode 100755 index 0000000..a947a46 --- /dev/null +++ b/pulsar-client-cpp/docker/build-wheels.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +set -e + +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR + +PYTHON_VERSIONS=( + '2.7 cp27-cp27mu' + '2.7 cp27-cp27m' + '3.3 cp33-cp33m' + '3.4 cp34-cp34m' + '3.5 cp35-cp35m' + '3.6 cp36-cp36m' +) + +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_SPEC=${PY[1]} + echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC" + + IMAGE_NAME=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC + + echo "Using image: $IMAGE_NAME" + docker run -i -v $PWD:/pulsar $IMAGE_NAME /build.sh +done diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh new file mode 100755 index 0000000..b27f6e9 --- /dev/null +++ b/pulsar-client-cpp/docker/create-images.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# Create all the Docker images for variations of Python versions + +set -e + +PYTHON_VERSIONS=( + '2.7 cp27-cp27mu' + '2.7 cp27-cp27m' + '3.3 cp33-cp33m' + '3.4 cp34-cp34m' + '3.5 cp35-cp35m' + '3.6 cp36-cp36m' +) + +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_SPEC=${PY[1]} + echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC" + + IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC + + docker build -t $IMAGE_NAME . \ + --build-arg PYTHON_VERSION=$PYTHON_VERSION \ + --build-arg PYTHON_SPEC=$PYTHON_SPEC + + echo "==== Successfully built image $IMAGE_NAME" +done diff --git a/pulsar-client-cpp/docker/push-images.sh b/pulsar-client-cpp/docker/push-images.sh new file mode 100755 index 0000000..5a542e0 --- /dev/null +++ b/pulsar-client-cpp/docker/push-images.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# Create all the Docker images for variations of Python versions + +set -e + +DOCKER_ORG=apachepulsar + +PYTHON_VERSIONS=( + '2.7 cp27-cp27mu' + '2.7 cp27-cp27m' + '3.3 cp33-cp33m' + '3.4 cp34-cp34m' + '3.5 cp35-cp35m' + '3.6 cp36-cp36m' +) + +for line in "${PYTHON_VERSIONS[@]}"; do + read -r -a PY <<< "$line" + PYTHON_VERSION=${PY[0]} + PYTHON_SPEC=${PY[1]} + + IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC + FULL_NAME=$DOCKER_ORG/$IMAGE_NAME + + echo "IMAGE_NAME: $IMAGE_NAME" + echo "FULL_NAME: $FULL_NAME" + docker tag $IMAGE_NAME $FULL_NAME + docker push $FULL_NAME + + echo "==== Successfully pushed image $FULL_NAME" +done -- To stop receiving notification emails like this one, please contact mme...@apache.org.