This is an automated email from the ASF dual-hosted git repository. xyz pushed a commit to branch branch-3.8 in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git
commit 922cc2ef515666071191f3d76ee3f1968a76f415 Author: Yunze Xu <[email protected]> AuthorDate: Fri Oct 10 09:43:31 2025 +0800 Use vcpkg to build debian packages (#506) (cherry picked from commit e04b2a983057332f656b9a356c217f0c8db42039) --- .github/workflows/ci-build-binary-artifacts.yaml | 1 - .github/workflows/ci-pr-validation.yaml | 1 - build-support/copy-deps-versionfile.sh | 29 ------- build-support/dep-version.py | 24 ------ dependencies.yaml | 29 ------- pkg/deb/Dockerfile | 101 ++--------------------- pkg/deb/build-deb.sh | 28 ++++++- pkg/deb/docker-build-deb-arm64.sh | 3 +- pkg/deb/docker-build-deb-x86_64.sh | 3 +- pkg/rpm/Dockerfile | 9 +- 10 files changed, 35 insertions(+), 193 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index f91e117..e45e2ed 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -56,7 +56,6 @@ jobs: run: build-support/generate-source-archive.sh - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build dependencies Docker image uses: docker/build-push-action@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 4efebfc..c73ec51 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -326,7 +326,6 @@ jobs: run: build-support/generate-source-archive.sh - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - name: Build dependencies Docker image uses: docker/build-push-action@v3 diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh deleted file mode 100755 index 8ecc8c0..0000000 --- a/build-support/copy-deps-versionfile.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) - -for dir in apk deb rpm; do - mkdir -p pkg/$dir/.build - cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build -done diff --git a/build-support/dep-version.py b/build-support/dep-version.py deleted file mode 100755 index 2421d42..0000000 --- a/build-support/dep-version.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import yaml, sys - -deps = yaml.safe_load(open('dependencies.yaml')) -print(deps[sys.argv[1]]) diff --git a/dependencies.yaml b/dependencies.yaml deleted file mode 100644 index 55087d2..0000000 --- a/dependencies.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Note: GCC 4.8 is incompatible with Boost >= 1.84 for the missed std::align -boost: 1.83.0 -cmake: 3.28.3 -protobuf: 3.20.0 -zlib: 1.3.1 -zstd: 1.5.5 -snappy: 1.1.10 -openssl: 1.1.1w -curl: 8.6.0 -roaring: 4.3.1 diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index bbafa8b..4d0212d 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -23,104 +23,15 @@ FROM debian:11 ARG PLATFORM -# perl is required to install OpenSSL RUN apt-get update -y && \ apt-get install -y \ curl \ g++ \ make \ - perl \ dpkg-dev \ - python3 \ - python3-pip \ - git - -RUN pip3 install pyyaml - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin - -# Download and compile boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ - tar zxf boost-${BOOST_VERSION}.tar.gz && \ - cd boost-${BOOST_VERSION} && \ - ./bootstrap.sh --with-libraries=regex && \ - ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} - -RUN CMAKE_VERSION=$(dep-version.py cmake) && \ - curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ - tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ - cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \ - cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \ - rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ - tar xfz v${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ - tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ - tar xfz ${SNAPPY_VERSION}.tar.gz && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -# Roaring -RUN ROARING_VERSION=$(dep-version.py roaring) && \ - curl -O -L https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v${ROARING_VERSION}.tar.gz && \ - tar xfz v${ROARING_VERSION}.tar.gz && \ - cd CRoaring-${ROARING_VERSION} && \ - mkdir build && cd build && CXXFLAGS="-fPIC -O3" cmake .. && \ - make -j8 && make install && \ - rm -rf /v${ROARING_VERSION}.tar.gz /CRoaring-${ROARING_VERSION} - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ - tar xfz curl-${CURL_VERSION}.tar.gz && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - - + git \ + zip \ + unzip \ + tar \ + pkg-config \ + ninja-build diff --git a/pkg/deb/build-deb.sh b/pkg/deb/build-deb.sh index 8415056..f466d02 100755 --- a/pkg/deb/build-deb.sh +++ b/pkg/deb/build-deb.sh @@ -20,6 +20,10 @@ set -e -x +if [[ $# -gt 0 ]]; then + CMAKE_ARCH=$1 +fi + cd /pulsar-client-cpp SRC_ROOT_DIR=$(pwd) cd pkg/deb @@ -37,12 +41,28 @@ cd BUILD tar xfz $SRC_ROOT_DIR/apache-pulsar-client-cpp-$POM_VERSION.tar.gz pushd $CPP_DIR -# link libraries for protoc -export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +git clone https://github.com/microsoft/vcpkg.git +# The cmake from debian:11 is 3.18, while vcpkg needs cmake >= 3.21 to build +if [[ $CMAKE_ARCH ]]; then + export CMAKE_VERSION=3.31.9 + curl -O -L https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-$CMAKE_ARCH.tar.gz + export PATH=$PWD/cmake-$CMAKE_VERSION-linux-$CMAKE_ARCH/bin/:$PATH + tar zxf cmake-*.tar.gz +fi +cmake --version chmod +x $(find . -name "*.sh") -cmake . -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF -DLINK_STATIC=ON -make -j 3 +if [[ $CMAKE_ARCH == "aarch64" ]]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 +fi +cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON +cmake --build build -j8 +./build-support/merge_archives_vcpkg.sh $PWD/build + +cp build/lib/libpulsar.a lib/libpulsar.a +cp build/lib/libpulsar.so lib/libpulsar.so +cp build/libpulsarwithdeps.a lib/libpulsarwithdeps.a popd DEST_DIR=apache-pulsar-client diff --git a/pkg/deb/docker-build-deb-arm64.sh b/pkg/deb/docker-build-deb-arm64.sh index f8c397c..99dced2 100755 --- a/pkg/deb/docker-build-deb-arm64.sh +++ b/pkg/deb/docker-build-deb-arm64.sh @@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=arm64 \ $IMAGE_NAME \ - /pulsar-client-cpp/pkg/deb/build-deb.sh + /pulsar-client-cpp/pkg/deb/build-deb.sh \ + aarch64 diff --git a/pkg/deb/docker-build-deb-x86_64.sh b/pkg/deb/docker-build-deb-x86_64.sh index fecf6a1..7537696 100755 --- a/pkg/deb/docker-build-deb-x86_64.sh +++ b/pkg/deb/docker-build-deb-x86_64.sh @@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=amd64 \ $IMAGE_NAME \ - /pulsar-client-cpp/pkg/deb/build-deb.sh + /pulsar-client-cpp/pkg/deb/build-deb.sh \ + x86_64 diff --git a/pkg/rpm/Dockerfile b/pkg/rpm/Dockerfile index 8fe564d..09635b4 100644 --- a/pkg/rpm/Dockerfile +++ b/pkg/rpm/Dockerfile @@ -30,16 +30,9 @@ RUN yum update -y && \ rpm-build \ which \ createrepo \ - git \ - python3 \ - python3-pip + git RUN dnf --enablerepo=powertools install -y libstdc++-static -RUN pip3 install pyyaml - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin - # Vcpkg does not provide pre-built binaries for Arm architectures so we need to build vcpkg from source RUN yum install -y cmake RUN dnf --enablerepo=devel install -y ninja-build
