This is an automated email from the ASF dual-hosted git repository. penghui pushed a commit to branch branch-2.8 in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 7aac8b806159a53a9f3e9f195fe2f2d214926934 Author: Livio Benčik <[email protected]> AuthorDate: Sun Jun 20 10:54:36 2021 +0200 [Issue 9888] add python3.9 on manylinux2014 build support (#10954) * add python3.9 on manylinux2014 build support * add python3.9 client build CI * fix python3.9 client build CI Co-authored-by: livio <[email protected]> (cherry picked from commit 825187f3b2785b3d2e940920ab1e8fc1db92906f) --- .github/workflows/ci-python-build-3.9-client.yaml | 62 ++++++++++++++++++++++ .../create-images.sh => docker-build-python3.9.sh} | 50 +++++++++-------- pulsar-client-cpp/docker/build-wheels.sh | 1 + pulsar-client-cpp/docker/create-images.sh | 17 +++--- .../docker/{ => manylinux1}/Dockerfile | 21 ++++---- .../docker/{ => manylinux2014}/Dockerfile | 23 ++++---- 6 files changed, 115 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml new file mode 100644 index 0000000..f8b04a7 --- /dev/null +++ b/.github/workflows/ci-python-build-3.9-client.yaml @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Python - Build 3.9 client +on: + pull_request: + branches: + - master + paths: + - '.github/workflows/**' + - 'pulsar-client-cpp/**' + push: + branches: + - branch-* + paths: + - '.github/workflows/**' + - 'pulsar-client-cpp/**' +jobs: + + build-wheel: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + + - name: Check changed files + id: check_changes + run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" + + - name: build python3.9 client + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: | + echo "Build Python3.9 client library" + pulsar-client-cpp/docker-build-python3.9.sh \ No newline at end of file diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker-build-python3.9.sh similarity index 52% copy from pulsar-client-cpp/docker/create-images.sh copy to pulsar-client-cpp/docker-build-python3.9.sh index 568f5f0..15cc2fd 100755 --- a/pulsar-client-cpp/docker/create-images.sh +++ b/pulsar-client-cpp/docker-build-python3.9.sh @@ -18,32 +18,30 @@ # under the License. # - -# Create all the Docker images for variations of Python versions +# Build Pulsar Python3.9 client set -e -PYTHON_VERSIONS=( - '2.7 cp27-cp27mu' - '2.7 cp27-cp27m' - '3.5 cp35-cp35m' - '3.6 cp36-cp36m' - '3.7 cp37-cp37m' - '3.8 cp38-cp38' - '3.9 cp39-cp39' -) - -for line in "${PYTHON_VERSIONS[@]}"; do - read -r -a PY <<< "$line" - PYTHON_VERSION=${PY[0]} - PYTHON_SPEC=${PY[1]} - echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC" - - IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC - - docker build -t $IMAGE_NAME . \ - --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg PYTHON_SPEC=$PYTHON_SPEC - - echo "==== Successfully built image $IMAGE_NAME" -done +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR/pulsar-client-cpp + + +# Build manylinux2014 build image +PYTHON_VERSION="3.9" +PYTHON_SPEC="cp39-cp39" +IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC + +docker build -t $IMAGE_NAME ./docker/manylinux2014 \ + --build-arg PYTHON_VERSION=$PYTHON_VERSION \ + --build-arg PYTHON_SPEC=$PYTHON_SPEC + + +# Build wheel file +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}" +IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC + +VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"} +COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh" +DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}" + +$DOCKER_CMD bash -c "${COMMAND}" diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh index 5371f85..a61f215 100755 --- a/pulsar-client-cpp/docker/build-wheels.sh +++ b/pulsar-client-cpp/docker/build-wheels.sh @@ -34,6 +34,7 @@ PYTHON_VERSIONS=( '3.6 cp36-cp36m' '3.7 cp37-cp37m' '3.8 cp38-cp38' + '3.9 cp39-cp39' ) function contains() { diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh index 568f5f0..6aa1d69 100755 --- a/pulsar-client-cpp/docker/create-images.sh +++ b/pulsar-client-cpp/docker/create-images.sh @@ -24,24 +24,25 @@ set -e PYTHON_VERSIONS=( - '2.7 cp27-cp27mu' - '2.7 cp27-cp27m' - '3.5 cp35-cp35m' - '3.6 cp36-cp36m' - '3.7 cp37-cp37m' - '3.8 cp38-cp38' - '3.9 cp39-cp39' + '2.7 cp27-cp27mu manylinux1' + '2.7 cp27-cp27m manylinux1' + '3.5 cp35-cp35m manylinux2014' + '3.6 cp36-cp36m manylinux2014' + '3.7 cp37-cp37m manylinux2014' + '3.8 cp38-cp38 manylinux2014' + '3.9 cp39-cp39 manylinux2014' ) for line in "${PYTHON_VERSIONS[@]}"; do read -r -a PY <<< "$line" PYTHON_VERSION=${PY[0]} PYTHON_SPEC=${PY[1]} + BASE_IMAGE=${PY[2]} echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC" IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC - docker build -t $IMAGE_NAME . \ + docker build -t $IMAGE_NAME $BASE_IMAGE \ --build-arg PYTHON_VERSION=$PYTHON_VERSION \ --build-arg PYTHON_SPEC=$PYTHON_SPEC diff --git a/pulsar-client-cpp/docker/Dockerfile b/pulsar-client-cpp/docker/manylinux1/Dockerfile similarity index 97% copy from pulsar-client-cpp/docker/Dockerfile copy to pulsar-client-cpp/docker/manylinux1/Dockerfile index 93f38b8..0df9240 100644 --- a/pulsar-client-cpp/docker/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux1/Dockerfile @@ -62,7 +62,7 @@ RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j # Download and compile boost -RUN curl -O -L https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz && \ +RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \ tar xvfz boost_1_68_0.tar.gz && \ cd /boost_1_68_0 && \ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \ @@ -126,14 +126,6 @@ RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \ make && make install && \ rm -rf /v3.12.1.tar.gz /CMake-3.12.1 -# LibCurl -RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ - tar xvfz curl-7.61.0.tar.gz && \ - cd curl-7.61.0 && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ - make && make install && \ - rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 - # Zstandard RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ tar xvfz zstd-1.3.7.tar.gz && \ @@ -150,6 +142,14 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1 make && make install && \ rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ + tar xvfz curl-7.61.0.tar.gz && \ + cd curl-7.61.0 && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ + make && make install && \ + rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + RUN pip install twine RUN pip install fastavro RUN pip install six @@ -159,6 +159,3 @@ RUN pip install enum34 ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -COPY build-wheel-file-within-docker.sh / -COPY build-client-lib-within-docker.sh / diff --git a/pulsar-client-cpp/docker/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile similarity index 96% rename from pulsar-client-cpp/docker/Dockerfile rename to pulsar-client-cpp/docker/manylinux2014/Dockerfile index 93f38b8..20a7247 100644 --- a/pulsar-client-cpp/docker/Dockerfile +++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile @@ -18,7 +18,7 @@ # -FROM quay.io/pypa/manylinux1_x86_64 +FROM quay.io/pypa/manylinux2014_x86_64 RUN yum install -y gtest-devel @@ -62,7 +62,7 @@ RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j # Download and compile boost -RUN curl -O -L https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz && \ +RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \ tar xvfz boost_1_68_0.tar.gz && \ cd /boost_1_68_0 && \ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \ @@ -126,14 +126,6 @@ RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \ make && make install && \ rm -rf /v3.12.1.tar.gz /CMake-3.12.1 -# LibCurl -RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ - tar xvfz curl-7.61.0.tar.gz && \ - cd curl-7.61.0 && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ - make && make install && \ - rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 - # Zstandard RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ tar xvfz zstd-1.3.7.tar.gz && \ @@ -150,6 +142,14 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1 make && make install && \ rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ + tar xvfz curl-7.61.0.tar.gz && \ + cd curl-7.61.0 && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ + make && make install && \ + rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + RUN pip install twine RUN pip install fastavro RUN pip install six @@ -159,6 +159,3 @@ RUN pip install enum34 ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -COPY build-wheel-file-within-docker.sh / -COPY build-client-lib-within-docker.sh /
