This is an automated email from the ASF dual-hosted git repository. kou pushed a commit to branch maint-6.0.x in repository https://gitbox.apache.org/repos/asf/arrow.git
commit daca509ffbc3e37faaaa47e2b16d3db142c05533 Author: Sutou Kouhei <[email protected]> AuthorDate: Wed Oct 27 05:28:49 2021 +0900 ARROW-14390: [Packaging][Ubuntu] Add support for Ubuntu 21.10 We need a workaround for aws-sdk-cpp with g++ 11 because Ubuntu 21.10 ships g++ 11. See also: https://github.com/aws/aws-sdk-cpp/issues/1750 Closes #11532 from kou/packaging-ubuntu-21.10 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]> --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 +++ dev/release/binary-task.rb | 2 +- dev/release/verify-release-candidate.sh | 19 +++-- dev/tasks/linux-packages/Rakefile | 4 ++ .../apt/ubuntu-impish/Dockerfile | 41 +++++++++++ .../apache-arrow/apt/ubuntu-impish-arm64/from | 18 +++++ .../apache-arrow/apt/ubuntu-impish/Dockerfile | 83 ++++++++++++++++++++++ dev/tasks/linux-packages/github.linux.amd64.yml | 4 ++ dev/tasks/linux-packages/package-task.rb | 10 ++- dev/tasks/tasks.yml | 3 +- 10 files changed, 181 insertions(+), 11 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 673a58e..a793f30 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3937,11 +3937,19 @@ macro(build_awssdk) DEPENDS aws_checksums_ep) add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) + set(AWSSDK_PATCH_COMMAND) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER + "10") + # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750 + set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g" + "<SOURCE_DIR>/cmake/compiler_settings.cmake") + endif() externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index f6a1619..5f88e47 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -1010,8 +1010,8 @@ class BinaryTask ["debian", "bookworm", "main"], ["ubuntu", "bionic", "main"], ["ubuntu", "focal", "main"], - ["ubuntu", "groovy", "main"], ["ubuntu", "hirsute", "main"], + ["ubuntu", "impish", "main"], ] end diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 8944397..3da8936 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -146,16 +146,21 @@ test_apt() { "ubuntu:focal" \ "arm64v8/ubuntu:focal" \ "ubuntu:hirsute" \ - "arm64v8/ubuntu:hirsute"; do \ + "arm64v8/ubuntu:hirsute" \ + "ubuntu:impish" \ + "arm64v8/ubuntu:impish"; do \ case "${target}" in - arm64v8/debian:bullseye|arm64v8/debian:bookworm|arm64v8/ubuntu:hirsute) - # qemu-user-static in Ubuntu 20.04 has a crash bug: - # https://bugs.launchpad.net/qemu/+bug/1749393 - continue - ;; arm64v8/*) if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then - : # OK + case "${target}" in + arm64v8/debian:buster|arm64v8/ubuntu:bionic|arm64v8/ubuntu:focal) + ;; # OK + *) + # qemu-user-static in Ubuntu 20.04 has a crash bug: + # https://bugs.launchpad.net/qemu/+bug/1749393 + continue + ;; + esac else continue fi diff --git a/dev/tasks/linux-packages/Rakefile b/dev/tasks/linux-packages/Rakefile index 6a67f28..58cd981 100644 --- a/dev/tasks/linux-packages/Rakefile +++ b/dev/tasks/linux-packages/Rakefile @@ -167,6 +167,8 @@ class LocalBinaryTask < BinaryTask # "ubuntu-focal-arm64", "ubuntu-hirsute", # "ubuntu-hirsute-arm64", + "ubuntu-impish", + # "ubuntu-impish-arm64", ] end @@ -211,6 +213,8 @@ class LocalBinaryTask < BinaryTask # Disable aarch64 targets by default for now # because they require some setups on host. [ + "almalinux-8", + # "almalinux-8-aarch64", "amazon-linux-2", # "amazon-linux-2-aarch64", "centos-7", diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile new file mode 100644 index 0000000..640cd71 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:impish + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from new file mode 100644 index 0000000..0c8a1c7 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:impish diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile new file mode 100644 index 0000000..2e94dd9 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:impish +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep -q '^libcuda1'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + else \ + :; \ + fi && \ + apt clean && \ + python3 -m pip install --no-use-pep517 meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index 9f35238..557c4ab 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -54,6 +54,10 @@ jobs: ARROW_VERSION: {{ arrow.version }} REPO: {{ '${{ secrets.REPO }}' }} YUM_TARGETS: {{ target }} + - uses: actions/upload-artifact@v2 + with: + name: packages + path: packages/*/{{ task_namespace }}/repositories/ - name: Docker Push continue-on-error: true shell: bash diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 6eb95f6..9dd1b2d 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -100,9 +100,9 @@ class PackageTask unless File.exist?(absolute_output_path) mkdir_p(File.dirname(absolute_output_path)) rake_output_message "Downloading... #{url}" - URI(url).open do |downloaded_file| + open_url(url) do |downloaded_file| File.open(absolute_output_path, "wb") do |output_file| - output_file.print(downloaded_file.read) + IO.copy_stream(downloaded_file, output_file) end end end @@ -110,6 +110,10 @@ class PackageTask absolute_output_path end + def open_url(url, &block) + URI(url).open(&block) + end + def substitute_content(content) content.gsub(/@(.+?)@/) do |matched| yield($1, matched) @@ -270,6 +274,8 @@ class PackageTask # "ubuntu-focal-arm64", "ubuntu-hirsute", # "ubuntu-hirsute-arm64", + "ubuntu-impish", + # "ubuntu-impish-arm64", ] end diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a307f97..9236722 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -446,7 +446,8 @@ tasks: "debian-bookworm", "ubuntu-bionic", "ubuntu-focal", - "ubuntu-hirsute"] %} + "ubuntu-hirsute", + "ubuntu-impish"] %} {% for architecture in ["amd64", "arm64"] %} {{ target }}-{{ architecture }}: {% if architecture == "amd64" %}
