This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new 6f5bebc ARROW-5520: [Packaging][deb] Add support for building on arm64 6f5bebc is described below commit 6f5bebc3411984dcedeee6b401fb16e9120f70b1 Author: Sutou Kouhei <k...@clear-code.com> AuthorDate: Mon Jun 17 12:07:49 2019 -0500 ARROW-5520: [Packaging][deb] Add support for building on arm64 We don't have NVIDIA CUDA toolkit and LLVM 7 on arm64. This includes Crossbow build configuration for arm64 but this is not used yet. Because building arm64 packages can't finish in 50min on Travis CI. Author: Sutou Kouhei <k...@clear-code.com> Closes #4588 from kou/packaging-deb-arm64 and squashes the following commits: f97736d63 <Sutou Kouhei> Add support for building on arm64 --- dev/tasks/linux-packages/apt/build.sh | 9 ++- .../apt/debian-stretch-arm64/Dockerfile | 77 ++++++++++++++++++++++ .../apt/debian-stretch-arm64/qemu-dummy-static | 33 ++++++++++ dev/tasks/linux-packages/debian/control | 40 +++++------ dev/tasks/linux-packages/debian/rules | 67 ++++++++++++------- dev/tasks/linux-packages/package-task.rb | 2 + .../{travis.linux.yml => travis.linux.arm64.yml} | 23 ++++++- dev/tasks/linux-packages/travis.linux.yml | 7 +- dev/tasks/tasks.yml | 44 ++++++++++++- 9 files changed, 246 insertions(+), 56 deletions(-) diff --git a/dev/tasks/linux-packages/apt/build.sh b/dev/tasks/linux-packages/apt/build.sh index b4a3fd6..a6a1cc4 100755 --- a/dev/tasks/linux-packages/apt/build.sh +++ b/dev/tasks/linux-packages/apt/build.sh @@ -41,7 +41,7 @@ case "${distribution}" in component=universe ;; esac -specific_debian_dir="debian.${distribution}-${code_name}" +architecture=$(dpkg-architecture -q DEB_BUILD_ARCH) run mkdir -p build run cp /host/tmp/${PACKAGE}-${VERSION}.tar.gz \ @@ -59,8 +59,11 @@ case "${VERSION}" in ;; esac run cd ${PACKAGE}-${VERSION}/ -if [ -d "/host/tmp/${specific_debian_dir}" ]; then - run cp -rp "/host/tmp/${specific_debian_dir}" debian +platform="${distribution}-${code_name}" +if [ -d "/host/tmp/debian.${platform}-${architecture}" ]; then + run cp -rp "/host/tmp/debian.${platform}-${architecture}" debian +elif [ -d "/host/tmp/debian.${platform}" ]; then + run cp -rp "/host/tmp/debian.${platform}" debian else run cp -rp "/host/tmp/debian" debian fi diff --git a/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile new file mode 100644 index 0000000..76365a0 --- /dev/null +++ b/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM arm64v8/debian:stretch + +COPY qemu-* /usr/bin/ + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +ARG DEBUG + +RUN \ + echo "deb http://deb.debian.org/debian stretch-backports main" > \ + /etc/apt/sources.list.d/backports.list + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + bison \ + build-essential \ + cmake \ + devscripts \ + flex \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libbrotli-dev \ + libc-ares-dev \ + libdouble-conversion-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgoogle-glog-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libzstd-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + python3-dev \ + python3-numpy \ + python3-pip \ + tzdata && \ + apt install -y -V -t stretch-backports ${quiet} \ + debhelper \ + libgmock-dev \ + libgrpc++-dev \ + libgtest-dev \ + libprotobuf-dev \ + libprotoc-dev \ + protobuf-compiler \ + protobuf-compiler-grpc \ + rapidjson-dev && \ + pip3 install --upgrade meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static b/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static new file mode 100755 index 0000000..82f0ea5 --- /dev/null +++ b/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Do nothing. This exists only for not requiring qemu-aarch64-static copy. +# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or +# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian +# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. +# +# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", +# the "COPY" is failed. It means that we always require "qemu*" even if we +# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, +# the "COPY" isn't failed. It means that we can copy "qemu*" only when we +# need. +# +# See also "script" in dev/tasks/linux-packages/travis.linux.yml. +# Travis CI uses old Ubuntu. So we need to put "qemu-aarch64-static" into +# this directory. diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control index 3febdc1..69e7fa6 100644 --- a/dev/tasks/linux-packages/debian/control +++ b/dev/tasks/linux-packages/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: Apache Arrow Developers <d...@arrow.apache.org> Build-Depends: bison, - clang-7, + clang-7 [!arm64], cmake, debhelper (>= 12), flex, @@ -25,7 +25,7 @@ Build-Depends: libssl-dev, libzstd-dev, ninja-build, - nvidia-cuda-toolkit, + nvidia-cuda-toolkit [!arm64], pkg-config, python3-dev, python3-numpy, @@ -63,7 +63,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libarrow-cuda14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -112,7 +112,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libarrow-cuda-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -136,7 +136,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libgandiva14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -150,7 +150,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: libgandiva-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -163,7 +163,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: libplasma14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -176,7 +176,7 @@ Description: Plasma is an in-memory object store and cache for big data. Package: plasma-store-server Section: utils -Architecture: any +Architecture: i386 amd64 Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, @@ -188,7 +188,7 @@ Description: Plasma is an in-memory object store and cache for big data. Package: libplasma-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -274,7 +274,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libarrow-cuda-glib14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -288,7 +288,7 @@ Description: Apache Arrow is a data processing library for analysis Package: gir1.2-arrow-cuda-1.0 Section: introspection -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${gir:Depends}, @@ -299,7 +299,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libarrow-cuda-glib-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -313,7 +313,7 @@ Description: Apache Arrow is a data processing library for analysis Package: libgandiva-glib14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -328,7 +328,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: gir1.2-gandiva-1.0 Section: introspection -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${gir:Depends}, @@ -340,7 +340,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: libgandiva-glib-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -355,7 +355,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: libgandiva-glib-doc Section: doc -Architecture: all +Architecture: i386 amd64 Multi-Arch: foreign Depends: ${misc:Depends} @@ -367,7 +367,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions Package: libplasma-glib14 Section: libs -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: @@ -381,7 +381,7 @@ Description: Plasma is an in-memory object store and cache for big data. Package: gir1.2-plasma-1.0 Section: introspection -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${gir:Depends}, @@ -392,7 +392,7 @@ Description: Plasma is an in-memory object store and cache for big data. Package: libplasma-glib-dev Section: libdevel -Architecture: any +Architecture: i386 amd64 Multi-Arch: same Depends: ${misc:Depends}, @@ -406,7 +406,7 @@ Description: Plasma is an in-memory object store and cache for big data. Package: libplasma-glib-doc Section: doc -Architecture: all +Architecture: i386 amd64 Multi-Arch: foreign Depends: ${misc:Depends} diff --git a/dev/tasks/linux-packages/debian/rules b/dev/tasks/linux-packages/debian/rules index efb0c55..32ea573 100755 --- a/dev/tasks/linux-packages/debian/rules +++ b/dev/tasks/linux-packages/debian/rules @@ -14,36 +14,53 @@ BUILD_TYPE=release dh $@ --with gir override_dh_auto_configure: - dh_auto_configure \ - --sourcedirectory=cpp \ - --builddirectory=cpp_build \ - --buildsystem=cmake+ninja \ - -- \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_CUDA=ON \ - -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=ON \ - -DARROW_GANDIVA_JAVA=OFF \ - -DARROW_ORC=ON \ - -DARROW_PARQUET=ON \ - -DARROW_PLASMA=ON \ - -DARROW_PYTHON=ON \ - -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ - -DPythonInterp_FIND_VERSION=ON \ + if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ + ARROW_CUDA=ON; \ + ARROW_PLASMA=ON; \ + else \ + ARROW_CUDA=OFF; \ + ARROW_PLASMA=OFF; \ + fi; \ + if dpkg -l clang-7 > /dev/null 2>&1; then \ + ARROW_GANDIVA=ON; \ + else \ + ARROW_GANDIVA=OFF; \ + fi; \ + if dpkg -l libprotobuf-dev > /dev/null 2>&1; then \ + cpp_build_system=cmake+ninja; \ + else \ + cpp_build_system=cmake; \ + fi; \ + dh_auto_configure \ + --sourcedirectory=cpp \ + --builddirectory=cpp_build \ + --buildsystem=$${cpp_build_system} \ + -- \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_CUDA=$${ARROW_CUDA} \ + -DARROW_FLIGHT=ON \ + -DARROW_GANDIVA=$${ARROW_GANDIVA} \ + -DARROW_GANDIVA_JAVA=OFF \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=$${ARROW_GANDIVA} \ + -DARROW_PYTHON=ON \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ + -DPythonInterp_FIND_VERSION=ON \ -DPythonInterp_FIND_VERSION_MAJOR=3 override_dh_auto_build: dh_auto_build \ --sourcedirectory=cpp \ - --builddirectory=cpp_build \ - --buildsystem=cmake+ninja - dh_auto_configure \ - --sourcedirectory=c_glib \ - --builddirectory=c_glib_build \ - --buildsystem=meson+ninja \ - -- \ - -Darrow_cpp_build_type=$(BUILD_TYPE) \ - -Darrow_cpp_build_dir=$(CURDIR)/cpp_build \ + --builddirectory=cpp_build + dh_auto_configure \ + --sourcedirectory=c_glib \ + --builddirectory=c_glib_build \ + --buildsystem=meson+ninja \ + -- \ + -Darrow_cpp_build_type=$(BUILD_TYPE) \ + -Darrow_cpp_build_dir=$(CURDIR)/cpp_build \ -Dgtk_doc=true env \ LD_LIBRARY_PATH=$(CURDIR)/cpp_build/$(BUILD_TYPE) \ diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 3f7fc1e..d1c08df 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -247,6 +247,8 @@ VERSION=#{@deb_upstream_version} if targets.empty? targets = [ "debian-stretch", + # Disable by default for now because it requires some setups on host. + # "debian-stretch-arm64", "ubuntu-xenial", "ubuntu-bionic", "ubuntu-cosmic", diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml similarity index 74% copy from dev/tasks/linux-packages/travis.linux.yml copy to dev/tasks/linux-packages/travis.linux.arm64.yml index c59ce08..853b706 100644 --- a/dev/tasks/linux-packages/travis.linux.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -16,16 +16,23 @@ # under the License. os: linux -dist: trusty -sudo: required +dist: xenial language: ruby # don't build twice if: tag IS blank +services: + - docker + +addons: + apt: + packages: + - qemu-user-static + - unar + env: global: - - PLAT=x86_64 - TRAVIS_TAG={{ task.tag }} - BUILD_REF={{ arrow.head }} - ARROW_VERSION={{ arrow.version }} @@ -39,6 +46,16 @@ script: - pushd arrow/dev/tasks/linux-packages - rake version:update - rake dist + # We can remove this with binfmt-support 2.1.7 or later and + # qemu-user-static 2.12 or later. It requires Debian buster or later, + # or Ubuntu 18.10 or later. + - wget http://archive.ubuntu.com/ubuntu/pool/universe/q/qemu/qemu-user-static_3.1+dfsg-2ubuntu5_amd64.deb + - unar *.deb + - rm *.deb + - pushd qemu-user-static* + - unar data.* + - cp data/usr/bin/qemu-aarch64-static ../apt/debian-stretch-arm64/ + - popd - {{ build_command }} deploy: diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml index c59ce08..937d4c7 100644 --- a/dev/tasks/linux-packages/travis.linux.yml +++ b/dev/tasks/linux-packages/travis.linux.yml @@ -16,16 +16,17 @@ # under the License. os: linux -dist: trusty -sudo: required +dist: xenial language: ruby +services: + - docker + # don't build twice if: tag IS blank env: global: - - PLAT=x86_64 - TRAVIS_TAG={{ task.tag }} - BUILD_REF={{ arrow.head }} - ARROW_VERSION={{ arrow.version }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 3e3acff..910650b 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -42,6 +42,8 @@ groups: - wheel-win-cp37m linux: - debian-stretch + # We can't finish this on Travis CI for now. + # - debian-stretch-arm64 - ubuntu-xenial - ubuntu-bionic - ubuntu-cosmic @@ -319,6 +321,44 @@ tasks: - plasma-store-server-dbgsym_{no_rc_version}-1_amd64.deb - plasma-store-server_{no_rc_version}-1_amd64.deb + # Don't use this for now. We can't complete this task in 50min on Travis CI. + debian-stretch-arm64: + platform: linux + template: linux-packages/travis.linux.arm64.yml + params: + build_command: rake apt:build APT_TARGETS=debian-stretch-arm64 + upload_extensions: + - .deb + - .dsc + - .debian.tar.xz + - .orig.tar.gz + artifacts: + - apache-arrow_{no_rc_version}-1.debian.tar.xz + - apache-arrow_{no_rc_version}-1.dsc + - apache-arrow_{no_rc_version}.orig.tar.gz + - gir1.2-arrow-1.0_{no_rc_version}-1_amd64.deb + - gir1.2-parquet-1.0_{no_rc_version}-1_amd64.deb + - libarrow-dev_{no_rc_version}-1_amd64.deb + - libarrow-flight-dev_{no_rc_version}-1_amd64.deb + - libarrow-flight14-dbgsym_{no_rc_version}-1_amd64.deb + - libarrow-flight14_{no_rc_version}-1_amd64.deb + - libarrow-glib-dev_{no_rc_version}-1_amd64.deb + - libarrow-glib-doc_{no_rc_version}-1_all.deb + - libarrow-glib14-dbgsym_{no_rc_version}-1_amd64.deb + - libarrow-glib14_{no_rc_version}-1_amd64.deb + - libarrow-python-dev_{no_rc_version}-1_amd64.deb + - libarrow-python14-dbgsym_{no_rc_version}-1_amd64.deb + - libarrow-python14_{no_rc_version}-1_amd64.deb + - libarrow14-dbgsym_{no_rc_version}-1_amd64.deb + - libarrow14_{no_rc_version}-1_amd64.deb + - libparquet-dev_{no_rc_version}-1_amd64.deb + - libparquet-glib-dev_{no_rc_version}-1_amd64.deb + - libparquet-glib-doc_{no_rc_version}-1_all.deb + - libparquet-glib14-dbgsym_{no_rc_version}-1_amd64.deb + - libparquet-glib14_{no_rc_version}-1_amd64.deb + - libparquet14-dbgsym_{no_rc_version}-1_amd64.deb + - libparquet14_{no_rc_version}-1_amd64.deb + ubuntu-xenial: platform: linux template: linux-packages/travis.linux.yml @@ -371,7 +411,7 @@ tasks: platform: linux template: linux-packages/travis.linux.yml params: - build_command: travis_wait 45 rake apt:build APT_TARGETS=ubuntu-bionic + build_command: rake apt:build APT_TARGETS=ubuntu-bionic upload_extensions: - .deb - .dsc @@ -422,7 +462,7 @@ tasks: platform: linux template: linux-packages/travis.linux.yml params: - build_command: travis_wait 45 rake apt:build APT_TARGETS=ubuntu-cosmic + build_command: rake apt:build APT_TARGETS=ubuntu-cosmic upload_extensions: - .deb - .dsc