This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cf61fa1 ARROW-10385: [C++][Gandiva] Add support for LLVM 11
cf61fa1 is described below
commit cf61fa1b8c1463f8f7c16e55c7275cac3111a0c2
Author: Sutou Kouhei <[email protected]>
AuthorDate: Tue Oct 27 16:02:42 2020 +0900
ARROW-10385: [C++][Gandiva] Add support for LLVM 11
LLVM link failures in MinGW build are caused by
https://github.com/msys2/MINGW-packages/issues/7170 .
They aren't fixed by this pull request. The problem will be fixed in
upstream.
Closes #8521 from kou/cpp-llvm-11
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.env | 4 ++--
.github/workflows/cpp.yml | 1 -
.github/workflows/cpp_cron.yml | 10 +++++-----
.github/workflows/python_cron.yml | 12 ++++++------
.github/workflows/ruby.yml | 1 -
.travis.yml | 6 ++++++
ci/conda_env_gandiva.yml | 4 ++--
...edora-32-cpp.dockerfile => fedora-33-cpp.dockerfile} | 16 ++++++++--------
ci/docker/linux-apt-c-glib.dockerfile | 1 +
ci/docker/ubuntu-20.04-cpp.dockerfile | 17 ++++++++++++++++-
ci/scripts/msys2_setup.sh | 6 +++---
cpp/CMakeLists.txt | 7 ++++++-
cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 +
cpp/src/gandiva/engine.cc | 1 +
dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 ++--
dev/tasks/gandiva-jars/build-cpp-linux.sh | 2 +-
dev/tasks/homebrew-formulae/apache-arrow.rb | 4 ++--
.../apache-arrow/apt/ubuntu-focal/Dockerfile | 9 +++++++++
dev/tasks/linux-packages/package-task.rb | 2 +-
dev/tasks/nuget-packages/github.linux.yml | 4 +++-
dev/tasks/tasks.yml | 8 ++++----
docker-compose.yml | 4 ++--
22 files changed, 81 insertions(+), 43 deletions(-)
diff --git a/.env b/.env
index c266213..5c6aaa5 100644
--- a/.env
+++ b/.env
@@ -24,9 +24,9 @@ ARCH=amd64
CUDA=9.1
DEBIAN=10
UBUNTU=18.04
-FEDORA=32
+FEDORA=33
PYTHON=3.6
-LLVM=10
+LLVM=11
CLANG_TOOLS=8
RUST=nightly-2020-04-22
GO=1.12
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 5ed1605..4e3e270 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -338,7 +338,6 @@ jobs:
CMAKE_ARGS: >-
-DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
-DBoost_NO_BOOST_CMAKE=ON
- CMAKE_GENERATOR: MSYS Makefiles
CMAKE_UNITY_BUILD: ON
steps:
- name: Disable Crash Dialogs
diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml
index 95c5f4b..5f404b2 100644
--- a/.github/workflows/cpp_cron.yml
+++ b/.github/workflows/cpp_cron.yml
@@ -46,7 +46,7 @@ jobs:
matrix:
name:
- amd64-debian-10-cpp
- - amd64-fedora-32-cpp
+ - amd64-fedora-33-cpp
- amd64-ubuntu-16.04-cpp
- amd64-ubuntu-18.04-cpp
- amd64-ubuntu-18.04-cpp-cmake32
@@ -55,10 +55,10 @@ jobs:
image: debian-cpp
title: AMD64 Debian 10 C++
debian: 10
- - name: amd64-fedora-32-cpp
+ - name: amd64-fedora-33-cpp
image: fedora-cpp
- title: AMD64 Fedora 32 C++
- fedora: 32
+ title: AMD64 Fedora 33 C++
+ fedora: 33
- name: amd64-ubuntu-16.04-cpp
image: ubuntu-cpp
title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
# the defaults here should correspond to the values in .env
ARCH: 'amd64'
DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 32 }}
+ FEDORA: ${{ matrix.fedora || 33 }}
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
steps:
- name: Checkout Arrow
diff --git a/.github/workflows/python_cron.yml
b/.github/workflows/python_cron.yml
index 133a73b..256856b 100644
--- a/.github/workflows/python_cron.yml
+++ b/.github/workflows/python_cron.yml
@@ -45,7 +45,7 @@ jobs:
matrix:
name:
- debian-10-python-3
- - fedora-32-python-3
+ - fedora-33-python-3
- ubuntu-18.04-python-3
- conda-python-3.7-dask-latest
- conda-python-3.7-turbodbc-latest
@@ -59,11 +59,11 @@ jobs:
image: debian-python
title: AMD64 Debian 10 Python 3
debian: 10
- - name: fedora-32-python-3
- cache: fedora-32-python-3
+ - name: fedora-33-python-3
+ cache: fedora-33-python-3
image: fedora-python
- title: AMD64 Fedora 32 Python 3
- fedora: 32
+ title: AMD64 Fedora 33 Python 3
+ fedora: 33
- name: ubuntu-18.04-python-3
cache: ubuntu-18.04-python-3
image: ubuntu-python
@@ -102,7 +102,7 @@ jobs:
env:
# the defaults here should correspond to the values in .env
DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 32 }}
+ FEDORA: ${{ matrix.fedora || 33 }}
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
PYTHON: ${{ matrix.python || 3.7 }}
HDFS: ${{ matrix.hdfs || '2.9.2' }}
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index d7c81bc..0558ac8 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -188,7 +188,6 @@ jobs:
CMAKE_ARGS: >-
-DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
-DBoost_NO_BOOST_CMAKE=ON
- CMAKE_GENERATOR: MSYS Makefiles
CMAKE_UNITY_BUILD: ON
steps:
- name: Disable Crash Dialogs
diff --git a/.travis.yml b/.travis.yml
index 72eed9b..84bbbce 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -65,6 +65,9 @@ jobs:
-e ARROW_GANDIVA=OFF
-e ARROW_PARQUET=OFF
"
+ # The LLVM's APT repository provides only arm64 binaries.
+ # We should use LLVM provided by Ubuntu.
+ LLVM: "10"
UBUNTU: "20.04"
- name: "C++ on s390x"
@@ -90,6 +93,9 @@ jobs:
-e cares_SOURCE=BUNDLED
-e gRPC_SOURCE=BUNDLED
"
+ # The LLVM's APT repository provides only arm64 binaries.
+ # We should use LLVM provided by Ubuntu.
+ LLVM: "10"
UBUNTU: "20.04"
- name: "Go on s390x"
diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml
index fa15d77..5056456 100644
--- a/ci/conda_env_gandiva.yml
+++ b/ci/conda_env_gandiva.yml
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-clangdev=10
-llvmdev=10
+clangdev=11
+llvmdev=11
re2
diff --git a/ci/docker/fedora-32-cpp.dockerfile
b/ci/docker/fedora-33-cpp.dockerfile
similarity index 92%
rename from ci/docker/fedora-32-cpp.dockerfile
rename to ci/docker/fedora-33-cpp.dockerfile
index 40fe461..9dde699 100644
--- a/ci/docker/fedora-32-cpp.dockerfile
+++ b/ci/docker/fedora-33-cpp.dockerfile
@@ -16,7 +16,7 @@
# under the License.
ARG arch
-FROM ${arch}/fedora:32
+FROM ${arch}/fedora:33
ARG arch
# install dependencies
@@ -32,17 +32,18 @@ RUN dnf update -y && \
cmake \
curl-devel \
flatbuffers-devel \
- java-1.8.0-openjdk-devel \
- java-1.8.0-openjdk-headless \
gcc \
gcc-c++ \
- glog-devel \
gflags-devel \
+ git \
+ glog-devel \
gmock-devel \
google-benchmark-devel \
- protobuf-devel \
+ grpc-devel \
+ grpc-plugins \
gtest-devel \
- git \
+ java-latest-openjdk-devel \
+ java-latest-openjdk-headless \
libzstd-devel \
llvm-devel \
llvm-static \
@@ -50,6 +51,7 @@ RUN dnf update -y && \
make \
ninja-build \
openssl-devel \
+ protobuf-devel \
python \
rapidjson-devel \
re2-devel \
@@ -64,7 +66,6 @@ COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
-# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck.
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_DATASET=ON \
@@ -85,7 +86,6 @@ ENV ARROW_BUILD_TESTS=ON \
AWSSDK_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
- gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PARQUET_BUILD_EXECUTABLES=ON \
PARQUET_BUILD_EXAMPLES=ON \
diff --git a/ci/docker/linux-apt-c-glib.dockerfile
b/ci/docker/linux-apt-c-glib.dockerfile
index dbd1304..12c6e23 100644
--- a/ci/docker/linux-apt-c-glib.dockerfile
+++ b/ci/docker/linux-apt-c-glib.dockerfile
@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
gtk-doc-tools \
libgirepository1.0-dev \
libglib2.0-doc \
+ lsb-release \
luarocks \
pkg-config \
ruby-dev && \
diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile
b/ci/docker/ubuntu-20.04-cpp.dockerfile
index d2a3fab..fbcda44 100644
--- a/ci/docker/ubuntu-20.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -30,7 +30,22 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
# while debugging package list with docker build.
ARG clang_tools
ARG llvm
-RUN apt-get update -y -q && \
+RUN if [ "${llvm}" -gt "10" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main"
> \
+ /etc/apt/sources.list.d/llvm.list && \
+ if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+ echo "deb https://apt.llvm.org/focal/
llvm-toolchain-focal-${clang_tools} main" > \
+ /etc/apt/sources.list.d/clang-tools.list; \
+ fi \
+ fi && \
+ apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
clang-${clang_tools} \
clang-${llvm} \
diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh
index 110b03b..51dc45f 100755
--- a/ci/scripts/msys2_setup.sh
+++ b/ci/scripts/msys2_setup.sh
@@ -24,26 +24,26 @@ target=$1
packages=()
case "${target}" in
cpp|c_glib|ruby)
- packages+=(make)
packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp)
- packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
packages+=(${MINGW_PACKAGE_PREFIX}-boost)
packages+=(${MINGW_PACKAGE_PREFIX}-brotli)
+ packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
packages+=(${MINGW_PACKAGE_PREFIX}-clang)
packages+=(${MINGW_PACKAGE_PREFIX}-cmake)
packages+=(${MINGW_PACKAGE_PREFIX}-gcc)
packages+=(${MINGW_PACKAGE_PREFIX}-gflags)
packages+=(${MINGW_PACKAGE_PREFIX}-grpc)
packages+=(${MINGW_PACKAGE_PREFIX}-gtest)
+ packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
packages+=(${MINGW_PACKAGE_PREFIX}-llvm)
packages+=(${MINGW_PACKAGE_PREFIX}-lz4)
+ packages+=(${MINGW_PACKAGE_PREFIX}-ninja)
packages+=(${MINGW_PACKAGE_PREFIX}-polly)
packages+=(${MINGW_PACKAGE_PREFIX}-protobuf)
packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy)
packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson)
packages+=(${MINGW_PACKAGE_PREFIX}-snappy)
packages+=(${MINGW_PACKAGE_PREFIX}-thrift)
- packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
packages+=(${MINGW_PACKAGE_PREFIX}-zlib)
packages+=(${MINGW_PACKAGE_PREFIX}-zstd)
;;
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d3c5b22..f9ab154 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -109,7 +109,12 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
-set(ARROW_LLVM_VERSIONS "10" "9" "8" "7")
+set(ARROW_LLVM_VERSIONS
+ "11"
+ "10"
+ "9"
+ "8"
+ "7")
list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY)
string(REGEX
REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index cc37a3c..d084f34 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1824,6 +1824,7 @@ macro(build_rapidjson)
set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include")
add_dependencies(toolchain rapidjson_ep)
+ add_dependencies(toolchain-tests rapidjson_ep)
add_dependencies(rapidjson rapidjson_ep)
set(RAPIDJSON_VENDORED TRUE)
diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc
index 735fd5f..a29a7e0 100644
--- a/cpp/src/gandiva/engine.cc
+++ b/cpp/src/gandiva/engine.cc
@@ -53,6 +53,7 @@
#include <llvm/Linker/Linker.h>
#include <llvm/MC/SubtargetFeature.h>
#include <llvm/Support/DynamicLibrary.h>
+#include <llvm/Support/Host.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_ostream.h>
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
index b5a2bec..272ed8c 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
@@ -68,8 +68,8 @@ outputs:
- glog
- grpc-cpp
- libprotobuf
- - clangdev 10
- - llvmdev 10
+ - clangdev 11
+ - llvmdev 11
- libutf8proc
- lz4-c
- numpy
diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh
b/dev/tasks/gandiva-jars/build-cpp-linux.sh
index 24402fc..20f9d32 100755
--- a/dev/tasks/gandiva-jars/build-cpp-linux.sh
+++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh
@@ -64,7 +64,7 @@ cmake -DCMAKE_BUILD_TYPE=Release \
-DBOOST_ROOT=/arrow_boost_dist \
-GNinja /arrow/cpp
ninja install
-ninja test
+CTEST_OUTPUT_ON_FAILURE=1 ninja test
popd
diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb
b/dev/tasks/homebrew-formulae/apache-arrow.rb
index 72d4cfa..888e2a3 100644
--- a/dev/tasks/homebrew-formulae/apache-arrow.rb
+++ b/dev/tasks/homebrew-formulae/apache-arrow.rb
@@ -16,7 +16,7 @@ class ApacheArrow < Formula
depends_on "numpy"
depends_on "[email protected]"
depends_on "protobuf"
- depends_on "[email protected]"
+ depends_on "[email protected]"
depends_on "rapidjson"
depends_on "snappy"
depends_on "thrift"
@@ -42,7 +42,7 @@ class ApacheArrow < Formula
-DARROW_WITH_SNAPPY=ON
-DARROW_WITH_BROTLI=ON
-DARROW_INSTALL_NAME_RPATH=OFF
- -DPYTHON_EXECUTABLE=#{Formula["[email protected]"].bin/"python3"}
+ -DPython3_EXECUTABLE=#{Formula["[email protected]"].bin/"python3"}
]
# Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in
ARROW-6437
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index 8715dd8..97f4412 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -34,6 +34,15 @@ RUN \
quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
apt update ${quiet} && \
apt install -y -V ${quiet} \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM} main" > \
+ /etc/apt/sources.list.d/llvm.list && \
+ apt update ${quiet} && \
+ apt install -y -V ${quiet} \
build-essential \
cmake \
debhelper \
diff --git a/dev/tasks/linux-packages/package-task.rb
b/dev/tasks/linux-packages/package-task.rb
index 1762e9f..7fa7af1 100644
--- a/dev/tasks/linux-packages/package-task.rb
+++ b/dev/tasks/linux-packages/package-task.rb
@@ -54,7 +54,7 @@ class PackageTask
@rpm_version = @version
@rpm_release = "1"
end
- @deb_release = "1"
+ @deb_release = ENV["DEB_RELEASE"] || "1"
end
def define
diff --git a/dev/tasks/nuget-packages/github.linux.yml
b/dev/tasks/nuget-packages/github.linux.yml
index c8143e4..95c4464 100644
--- a/dev/tasks/nuget-packages/github.linux.yml
+++ b/dev/tasks/nuget-packages/github.linux.yml
@@ -40,7 +40,9 @@ jobs:
git -C arrow checkout FETCH_HEAD
git -C arrow submodule update --init --recursive
- name: Setup Python
- uses: actions/setup-python@v1
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
- name: Setup Archery
run: pip install -e arrow/dev/archery[docker]
- name: Prepare version
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 581e174..9cda368 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1599,12 +1599,12 @@ tasks:
UBUNTU: 18.04
run: ubuntu-cpp
- test-fedora-32-cpp:
+ test-fedora-33-cpp:
ci: circle
template: docker-tests/circle.linux.yml
params:
env:
- FEDORA: 32
+ FEDORA: 33
run: fedora-cpp
test-ubuntu-18.04-cpp-release:
@@ -1719,12 +1719,12 @@ tasks:
UBUNTU: 18.04
run: ubuntu-python
- test-fedora-32-python-3:
+ test-fedora-33-python-3:
ci: azure
template: docker-tests/azure.linux.yml
params:
env:
- FEDORA: 32
+ FEDORA: 33
run: fedora-python
test-r-linux-as-cran:
diff --git a/docker-compose.yml b/docker-compose.yml
index 90fdb97..d130ebc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -332,7 +332,7 @@ services:
# docker-compose run --rm fedora-cpp
# Parameters:
# ARCH: amd64, arm64v8, ...
- # FEDORA: 32
+ # FEDORA: 33
image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
build:
context: .
@@ -572,7 +572,7 @@ services:
# docker-compose run --rm fedora-python
# Parameters:
# ARCH: amd64, arm64v8, ...
- # FEDORA: 32
+ # FEDORA: 33
image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
build:
context: .
