This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-java.git
The following commit(s) were added to refs/heads/main by this push:
new aee890f9 GH-527: [Release] Add support for .jar for Windows (#528)
aee890f9 is described below
commit aee890f967e951e82ae95fbd43eb6aed4e0d7483
Author: Sutou Kouhei <[email protected]>
AuthorDate: Fri Jan 17 14:51:37 2025 +0900
GH-527: [Release] Add support for .jar for Windows (#528)
Fixes GH-527.
It seems that arrow-testing with ARROW_BUILD_STATIC=ON and
GTest_SOURCE=BUNDLED is broken. We'll enable JNI test after we fix it in
apache/arrow.
---
.github/workflows/rc.yml | 93 +++++++++++++++--
.pre-commit-config.yaml | 2 +
ci/scripts/download_cpp.sh | 2 +-
ci/scripts/jni_build.sh | 17 ++-
ci/scripts/jni_macos_build.sh | 36 ++++---
ci/scripts/jni_manylinux_build.sh | 52 ++++++----
.../{jni_macos_build.sh => jni_windows_build.sh} | 114 ++++++++++-----------
ci/scripts/util_log.sh | 28 +++++
8 files changed, 234 insertions(+), 110 deletions(-)
diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
index 60a8a61d..4c206625 100644
--- a/.github/workflows/rc.yml
+++ b/.github/workflows/rc.yml
@@ -76,7 +76,7 @@ jobs:
name: release-source
path: |
apache-arrow-java-*
- jni-ubuntu:
+ jni-linux:
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
runs-on: ${{ matrix.platform.runs_on }}
needs:
@@ -100,7 +100,7 @@ jobs:
uses:
actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-source
- - name: Extract Download the latest Apache Arrow C++
+ - name: Extract source archive
run: |
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
- name: Download the latest Apache Arrow C++
@@ -127,7 +127,7 @@ jobs:
path: .docker
key: jni-linux-${{ matrix.platform.arch }}-${{
hashFiles('arrow/cpp/**') }}
restore-keys: jni-linux-${{ matrix.platform.arch }}-
- - name: Build C++ libraries
+ - name: Build
run: |
docker compose run vcpkg-jni
- name: Push Docker image
@@ -159,7 +159,7 @@ jobs:
uses:
actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-source
- - name: Extract Download the latest Apache Arrow C++
+ - name: Extract source archive
run: |
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
- name: Download the latest Apache Arrow C++
@@ -176,7 +176,7 @@ jobs:
repository: apache/parquet-testing
path: arrow/cpp/submodules/parquet-testing
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #
v5.3.0
with:
cache: 'pip'
python-version: 3.12
@@ -246,7 +246,7 @@ jobs:
path: ccache
key: jni-macos-${{ matrix.platform.arch }}-${{
hashFiles('arrow/cpp/**') }}
restore-keys: jni-macos-${{ matrix.platform.arch }}-
- - name: Build C++ libraries
+ - name: Build
run: |
set -e
# make brew Java available to CMake
@@ -259,12 +259,83 @@ jobs:
with:
name: jni-macos-${{ matrix.platform.arch }}
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
+ jni-windows:
+ name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
+ runs-on: ${{ matrix.platform.runs_on }}
+ needs:
+ - source
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - runs_on: windows-2019
+ arch: "x86_64"
+ steps:
+ - name: Download source archive
+ uses:
actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+ with:
+ name: release-source
+ - name: Extract source archive
+ shell: bash
+ run: |
+ tar -xf apache-arrow-java-*.tar.gz --strip-components=1
+ - name: Download the latest Apache Arrow C++
+ shell: bash
+ run: |
+ ci/scripts/download_cpp.sh
+ - name: Set up Java
+ uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b #
v4.6.0
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ - name: Download Timezone Database
+ shell: bash
+ run: |
+ arrow/ci/scripts/download_tz_database.sh
+ - name: Install ccache
+ shell: bash
+ run: |
+ env | sort
+ version=4.10.2
+ base_name="ccache-${version}-windows-x86_64"
+
url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip";
+ curl --fail --location --remote-name "${url}"
+ unzip "${base_name}.zip"
+ chmod +x "${base_name}/ccache.exe"
+ mv "${base_name}/ccache.exe" /usr/bin/
+ rm -rf "${base_name}"{,.zip}
+ - name: Prepare ccache
+ shell: bash
+ run: |
+ echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
+ - name: Cache ccache
+ uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+ with:
+ path: ccache
+ key: jni-windows-${{ matrix.platform.arch }}-${{
hashFiles('arrow/cpp/**') }}
+ restore-keys: jni-windows-${{ matrix.platform.arch }}-
+ - name: Build
+ shell: cmd
+ run: |
+ call "C:\Program Files (x86)\Microsoft Visual
Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+ REM For ORC
+ set TZDIR=/c/msys64/usr/share/zoneinfo
+ bash -c "ci/scripts/jni_windows_build.sh . arrow build jni"
+ - name: Compress into single artifact to keep directory structure
+ shell: bash
+ run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/
+ - name: Upload artifacts
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08
# v4.6.0
+ with:
+ name: jni-windows-${{ matrix.platform.arch }}
+ path: jni-windows-${{ matrix.platform.arch }}.tar.gz
binaries:
name: Binaries
runs-on: ubuntu-latest
needs:
- - jni-ubuntu
+ - jni-linux
- jni-macos
+ - jni-windows
steps:
- name: Download artifacts
uses:
actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -278,7 +349,7 @@ jobs:
# tar -xvzf jni-linux-aarch_64.tar.gz
tar -xvzf jni-macos-x86_64.tar.gz
tar -xvzf jni-macos-aarch_64.tar.gz
- # tar -xvzf jni-windows.tar.gz
+ tar -xvzf jni-windows-x86_64.tar.gz
- name: Test that shared libraries exist
run: |
set -x
@@ -303,9 +374,9 @@ jobs:
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
- # test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
- # test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
- # test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
+ test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
+ test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
+ test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
- name: Checkout apache/arrow-testing
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
with:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0a762414..ca3cacec 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -47,6 +47,8 @@ repos:
rev: v0.10.0
hooks:
- id: shellcheck
+ args:
+ - "--external-sources"
- repo: https://github.com/scop/pre-commit-shfmt
rev: v3.9.0-1
hooks:
diff --git a/ci/scripts/download_cpp.sh b/ci/scripts/download_cpp.sh
index 4d801a47..3721f62a 100755
--- a/ci/scripts/download_cpp.sh
+++ b/ci/scripts/download_cpp.sh
@@ -47,6 +47,6 @@ fi
if [ -z "${url}" ]; then
url="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
fi
-wget --output-document "apache-arrow-${version}.tar.gz" "${url}"
+curl --location --output "apache-arrow-${version}.tar.gz" "${url}"
tar xf "apache-arrow-${version}.tar.gz"
mv "apache-arrow-${version}" arrow
diff --git a/ci/scripts/jni_build.sh b/ci/scripts/jni_build.sh
index a77b0da0..aec6fc32 100755
--- a/ci/scripts/jni_build.sh
+++ b/ci/scripts/jni_build.sh
@@ -17,20 +17,25 @@
# specific language governing permissions and limitations
# under the License.
-set -euxo pipefail
+set -euo pipefail
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
source_dir=${1}
arrow_install_dir=${2}
build_dir=${3}/java_jni
# The directory where the final binaries will be stored when scripts finish
dist_dir=${4}
prefix_dir="${build_dir}/java-jni"
+github_actions_group_end
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
+github_actions_group_end
-echo "=== Building Arrow Java C Data Interface native library ==="
+github_actions_group_begin "Building Arrow Java C Data Interface native
library"
case "$(uname)" in
Linux)
@@ -71,6 +76,9 @@ if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then
fi
cmake --build "${build_dir}" --target install
+github_actions_group_end
+
+github_actions_group_begin "Copying artifacts"
mkdir -p "${dist_dir}"
# For Windows. *.dll are installed into bin/ on Windows.
if [ -d "${prefix_dir}/bin" ]; then
@@ -78,3 +86,4 @@ if [ -d "${prefix_dir}/bin" ]; then
else
mv "${prefix_dir}"/lib/* "${dist_dir}"/
fi
+github_actions_group_end
diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh
index 5a693031..1ad76266 100755
--- a/ci/scripts/jni_macos_build.sh
+++ b/ci/scripts/jni_macos_build.sh
@@ -20,8 +20,12 @@
# This script is like java_jni_build.sh, but is meant for release artifacts
# and hardcodes assumptions about the environment it is being run in.
-set -ex
+set -euo pipefail
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
source_dir="$(cd "${1}" && pwd)"
arrow_dir="$(cd "${2}" && pwd)"
build_dir="${3}"
@@ -36,20 +40,29 @@ i386)
esac
# The directory where the final binaries will be stored when scripts finish
dist_dir="${4}"
+github_actions_group_end
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
rm -rf "${dist_dir}"
mkdir -p "${build_dir}"
build_dir="$(cd "${build_dir}" && pwd)"
+github_actions_group_end
+
+: "${ARROW_USE_CCACHE:=ON}"
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+ github_actions_group_begin "ccache statistics before build"
+ ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
+fi
-echo "=== Building Arrow C++ libraries ==="
+github_actions_group_begin "Building Arrow C++ libraries"
install_dir="${build_dir}/cpp-install"
: "${ARROW_ACERO:=ON}"
export ARROW_ACERO
: "${ARROW_BUILD_TESTS:=ON}"
+export ARROW_BUILD_TESTS
: "${ARROW_DATASET:=ON}"
export ARROW_DATASET
: "${ARROW_GANDIVA:=ON}"
@@ -58,15 +71,9 @@ export ARROW_GANDIVA
export ARROW_ORC
: "${ARROW_PARQUET:=ON}"
: "${ARROW_S3:=ON}"
-: "${ARROW_USE_CCACHE:=ON}"
: "${CMAKE_BUILD_TYPE:=Release}"
: "${CMAKE_UNITY_BUILD:=ON}"
-if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics before build ==="
- ccache -sv 2>/dev/null || ccache -s
-fi
-
export ARROW_TEST_DATA="${arrow_dir}/testing/data"
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
export AWS_EC2_METADATA_DISABLED=TRUE
@@ -99,8 +106,10 @@ cmake \
-Dre2_SOURCE=BUNDLED \
-GNinja
cmake --build "${build_dir}/cpp" --target install
+github_actions_group_end
if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
+ github_actions_group_begin "Running Arrow C++ libraries tests"
# MinIO is required
exclude_tests="arrow-s3fs-test"
# unstable
@@ -113,6 +122,7 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
--parallel "$(sysctl -n hw.ncpu)" \
--test-dir "${build_dir}/cpp" \
--timeout 300
+ github_actions_group_end
fi
export
JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
@@ -123,11 +133,12 @@ export
JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install
"${dist_dir}"
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics after build ==="
+ github_actions_group_begin "ccache statistics after build"
ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
fi
-echo "=== Checking shared dependencies for libraries ==="
+github_actions_group_begin "Checking shared dependencies for libraries"
pushd "${dist_dir}"
archery linking check-dependencies \
--allow CoreFoundation \
@@ -147,3 +158,4 @@ archery linking check-dependencies \
"arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \
"gandiva_jni/${normalized_arch}/libgandiva_jni.dylib"
popd
+github_actions_group_end
diff --git a/ci/scripts/jni_manylinux_build.sh
b/ci/scripts/jni_manylinux_build.sh
index 0251c71c..148d2e02 100755
--- a/ci/scripts/jni_manylinux_build.sh
+++ b/ci/scripts/jni_manylinux_build.sh
@@ -20,37 +20,45 @@
# This script is like java_jni_build.sh, but is meant for release artifacts
# and hardcodes assumptions about the environment it is being run in.
-set -exo pipefail
+set -euo pipefail
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
source_dir="$(cd "${1}" && pwd)"
arrow_dir="$(cd "${2}" && pwd)"
build_dir="${3}"
-normalized_arch="$(arch)"
-case "${normalized_arch}" in
-aarch64)
- normalized_arch=aarch_64
- ;;
-esac
# The directory where the final binaries will be stored when scripts finish
dist_dir="${4}"
+github_actions_group_end
-echo "=== Install Archery ==="
+github_actions_group_begin "Install Archery"
pip install -e "${arrow_dir}/dev/archery[all]"
+github_actions_group_end
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
rm -rf "${dist_dir}"
mkdir -p "${build_dir}"
build_dir="$(cd "${build_dir}" && pwd)"
+github_actions_group_end
+
+: "${ARROW_USE_CCACHE:=ON}"
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+ github_actions_group_begin "ccache statistics before build"
+ ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
+fi
-echo "=== Building Arrow C++ libraries ==="
+github_actions_group_begin "Building Arrow C++ libraries"
devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' |
grep -o "^[0-9]*")"
devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
: "${ARROW_ACERO:=ON}"
export ARROW_ACERO
: "${ARROW_BUILD_TESTS:=ON}"
+export ARROW_BUILD_TESTS
: "${ARROW_DATASET:=ON}"
export ARROW_DATASET
: "${ARROW_GANDIVA:=ON}"
@@ -63,7 +71,6 @@ export ARROW_GANDIVA
export ARROW_ORC
: "${ARROW_PARQUET:=ON}"
: "${ARROW_S3:=ON}"
-: "${ARROW_USE_CCACHE:=ON}"
: "${CMAKE_BUILD_TYPE:=release}"
: "${CMAKE_UNITY_BUILD:=ON}"
: "${VCPKG_ROOT:=/opt/vcpkg}"
@@ -71,16 +78,10 @@ export ARROW_ORC
:
"${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}"
:
"${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}"
-if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics before build ==="
- ccache -sv 2>/dev/null || ccache -s
-fi
-
export ARROW_TEST_DATA="${arrow_dir}/testing/data"
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
export AWS_EC2_METADATA_DISABLED=TRUE
-mkdir -p "${build_dir}/cpp"
install_dir="${build_dir}/cpp-install"
cmake \
@@ -119,8 +120,10 @@ cmake \
-GNinja
cmake --build "${build_dir}/cpp"
cmake --install "${build_dir}/cpp"
+github_actions_group_end
if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then
+ github_actions_group_begin "Running Arrow C++ libraries tests"
# MinIO is required
exclude_tests="arrow-s3fs-test"
case $(arch) in
@@ -146,6 +149,7 @@ if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then
--parallel "$(nproc)" \
--test-dir "${build_dir}/cpp" \
--timeout 300
+ github_actions_group_end
fi
JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
@@ -158,11 +162,18 @@ export JAVA_JNI_CMAKE_ARGS
"${dist_dir}"
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics after build ==="
+ github_actions_group_begin "ccache statistics after build"
ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
fi
-echo "=== Checking shared dependencies for libraries ==="
+github_actions_group_begin "Checking shared dependencies for libraries"
+normalized_arch="$(arch)"
+case "${normalized_arch}" in
+aarch64)
+ normalized_arch=aarch_64
+ ;;
+esac
pushd "${dist_dir}"
archery linking check-dependencies \
--allow ld-linux-aarch64 \
@@ -181,3 +192,4 @@ archery linking check-dependencies \
arrow_orc_jni/"${normalized_arch}"/libarrow_orc_jni.so \
gandiva_jni/"${normalized_arch}"/libgandiva_jni.so
popd
+github_actions_group_end
diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_windows_build.sh
similarity index 58%
copy from ci/scripts/jni_macos_build.sh
copy to ci/scripts/jni_windows_build.sh
index 5a693031..d01ef45f 100755
--- a/ci/scripts/jni_macos_build.sh
+++ b/ci/scripts/jni_windows_build.sh
@@ -17,56 +17,48 @@
# specific language governing permissions and limitations
# under the License.
-# This script is like java_jni_build.sh, but is meant for release artifacts
-# and hardcodes assumptions about the environment it is being run in.
+set -euo pipefail
-set -ex
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+github_actions_group_begin "Prepare arguments"
source_dir="$(cd "${1}" && pwd)"
arrow_dir="$(cd "${2}" && pwd)"
build_dir="${3}"
-normalized_arch="$(arch)"
-case "${normalized_arch}" in
-arm64)
- normalized_arch=aarch_64
- ;;
-i386)
- normalized_arch=x86_64
- ;;
-esac
# The directory where the final binaries will be stored when scripts finish
dist_dir="${4}"
+github_actions_group_end
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
rm -rf "${build_dir}"
-rm -rf "${dist_dir}"
mkdir -p "${build_dir}"
build_dir="$(cd "${build_dir}" && pwd)"
+github_actions_group_end
+
+: "${ARROW_USE_CCACHE:=ON}"
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+ github_actions_group_begin "ccache statistics before build"
+ ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
+fi
-echo "=== Building Arrow C++ libraries ==="
+github_actions_group_begin "Building Arrow C++ libraries"
install_dir="${build_dir}/cpp-install"
: "${ARROW_ACERO:=ON}"
export ARROW_ACERO
-: "${ARROW_BUILD_TESTS:=ON}"
+: "${ARROW_BUILD_TESTS:=OFF}" # TODO: ON
+export ARROW_BUILD_TESTS
: "${ARROW_DATASET:=ON}"
export ARROW_DATASET
-: "${ARROW_GANDIVA:=ON}"
-export ARROW_GANDIVA
: "${ARROW_ORC:=ON}"
export ARROW_ORC
: "${ARROW_PARQUET:=ON}"
: "${ARROW_S3:=ON}"
-: "${ARROW_USE_CCACHE:=ON}"
-: "${CMAKE_BUILD_TYPE:=Release}"
+: "${CMAKE_BUILD_TYPE:=release}"
: "${CMAKE_UNITY_BUILD:=ON}"
-if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics before build ==="
- ccache -sv 2>/dev/null || ccache -s
-fi
-
export ARROW_TEST_DATA="${arrow_dir}/testing/data"
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
export AWS_EC2_METADATA_DISABLED=TRUE
@@ -76,46 +68,45 @@ cmake \
-B "${build_dir}/cpp" \
-DARROW_ACERO="${ARROW_ACERO}" \
-DARROW_BUILD_SHARED=OFF \
- -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \
+ -DARROW_BUILD_TESTS=ON \
-DARROW_CSV="${ARROW_DATASET}" \
-DARROW_DATASET="${ARROW_DATASET}" \
-DARROW_SUBSTRAIT="${ARROW_DATASET}" \
-DARROW_DEPENDENCY_USE_SHARED=OFF \
- -DARROW_GANDIVA="${ARROW_GANDIVA}" \
- -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \
- -DARROW_JSON="${ARROW_DATASET}" \
-DARROW_ORC="${ARROW_ORC}" \
-DARROW_PARQUET="${ARROW_PARQUET}" \
-DARROW_S3="${ARROW_S3}" \
-DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \
- -DAWSSDK_SOURCE=BUNDLED \
+ -DARROW_WITH_BROTLI=ON \
+ -DARROW_WITH_LZ4=ON \
+ -DARROW_WITH_SNAPPY=ON \
+ -DARROW_WITH_ZSTD=ON \
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
-DCMAKE_INSTALL_PREFIX="${install_dir}" \
-DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \
- -DGTest_SOURCE=BUNDLED \
- -DPARQUET_BUILD_EXAMPLES=OFF \
- -DPARQUET_BUILD_EXECUTABLES=OFF \
- -DPARQUET_REQUIRE_ENCRYPTION=OFF \
- -Dre2_SOURCE=BUNDLED \
-GNinja
-cmake --build "${build_dir}/cpp" --target install
+cmake --build "${build_dir}/cpp"
+cmake --install "${build_dir}/cpp"
+github_actions_group_end
-if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
+if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then
+ github_actions_group_begin "Running Arrow C++ libraries tests"
# MinIO is required
exclude_tests="arrow-s3fs-test"
# unstable
- exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
- exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
+ exclude_tests="${exclude_tests}|arrow-compute-hash-join-node-test"
+ exclude_tests="${exclude_tests}|arrow-dataset-scanner-test"
+ # strptime
+ exclude_tests="${exclude_tests}|arrow-utility-test"
ctest \
--exclude-regex "${exclude_tests}" \
--label-regex unittest \
--output-on-failure \
- --parallel "$(sysctl -n hw.ncpu)" \
- --test-dir "${build_dir}/cpp" \
+ --parallel "$(nproc)" \
--timeout 300
+ github_actions_group_end
fi
-export
JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
"${source_dir}/ci/scripts/jni_build.sh" \
"${source_dir}" \
"${install_dir}" \
@@ -123,27 +114,26 @@ export
JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install
"${dist_dir}"
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
- echo "=== ccache statistics after build ==="
+ github_actions_group_begin "ccache statistics after build"
ccache -sv 2>/dev/null || ccache -s
+ github_actions_group_end
fi
-echo "=== Checking shared dependencies for libraries ==="
+github_actions_group_begin "Checking shared dependencies for libraries"
+normalized_arch="$(arch)"
+case "${normalized_arch}" in
+aarch64)
+ normalized_arch=aarch_64
+ ;;
+esac
pushd "${dist_dir}"
-archery linking check-dependencies \
- --allow CoreFoundation \
- --allow Security \
- --allow libSystem \
- --allow libarrow_cdata_jni \
- --allow libarrow_dataset_jni \
- --allow libarrow_orc_jni \
- --allow libc++ \
- --allow libcurl \
- --allow libgandiva_jni \
- --allow libncurses \
- --allow libobjc \
- --allow libz \
- "arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib" \
- "arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib" \
- "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \
- "gandiva_jni/${normalized_arch}/libgandiva_jni.dylib"
+# TODO
+# archery linking check-dependencies \
+# --allow libm \
+# --allow librt \
+# --allow libz \
+# arrow_cdata_jni/"${normalized_arch}"/libarrow_cdata_jni.dll \
+# arrow_dataset_jni/"${normalized_arch}"/libarrow_dataset_jni.dll \
+# arrow_orc_jni/"${normalized_arch}"/libarrow_orc_jni.dll
popd
+github_actions_group_end
diff --git a/ci/scripts/util_log.sh b/ci/scripts/util_log.sh
new file mode 100644
index 00000000..c8ee48bb
--- /dev/null
+++ b/ci/scripts/util_log.sh
@@ -0,0 +1,28 @@
+# shellcheck shell=bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+github_actions_group_begin() {
+ echo "::group::$1"
+ set -x
+}
+
+github_actions_group_end() {
+ set +x
+ echo "::endgroup::"
+}
