This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-java.git
The following commit(s) were added to refs/heads/main by this push:
new d650aa03 GH-14: Add JNI CI test (#441)
d650aa03 is described below
commit d650aa03229350e3c7edfc55defd305585fc9bc8
Author: David Li <[email protected]>
AuthorDate: Wed Dec 4 01:28:21 2024 -0500
GH-14: Add JNI CI test (#441)
Fixes #14.
---
.env | 2 +-
.github/workflows/test.yml | 11 ++++--
ci/docker/conda-jni.dockerfile | 30 ++++++++++++++++
ci/scripts/java_jni_build.sh | 80 ++++++++++++++++++++++++++++++++++++++++++
docker-compose.yml | 42 ++++++++++++++++++----
5 files changed, 155 insertions(+), 10 deletions(-)
diff --git a/.env b/.env
index bb9f63ab..b50a16eb 100644
--- a/.env
+++ b/.env
@@ -47,4 +47,4 @@ ULIMIT_CORE=-1
# Default versions for various dependencies
JDK=11
-MAVEN=3.9.6
+MAVEN=3.9.9
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 473ce84c..69c8db0f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -38,7 +38,7 @@ env:
jobs:
ubuntu:
- name: AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
+ name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{
matrix.maven }}
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 30
@@ -46,8 +46,13 @@ jobs:
fail-fast: false
matrix:
jdk: [11, 17, 21, 22]
- maven: [3.9.6]
- image: [java]
+ maven: [3.9.9]
+ image: [ubuntu, conda-jni-cdata]
+ include:
+ - image: ubuntu
+ name: "Ubuntu"
+ - image: conda-jni-cdata
+ name: "Conda JNI"
env:
JDK: ${{ matrix.jdk }}
MAVEN: ${{ matrix.maven }}
diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile
new file mode 100644
index 00000000..e14db736
--- /dev/null
+++ b/ci/docker/conda-jni.dockerfile
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ghcr.io/mamba-org/micromamba:ubuntu24.04
+
+ARG jdk=11
+ARG maven=3.9.9
+
+RUN micromamba install -y \
+ -c conda-forge \
+ cmake \
+ compilers \
+ maven=${maven} \
+ ninja \
+ openjdk=${jdk} && \
+ micromamba clean --all
diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh
new file mode 100755
index 00000000..7b63dbee
--- /dev/null
+++ b/ci/scripts/java_jni_build.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eo pipefail
+
+arrow_dir=${1}
+arrow_install_dir=${2}
+build_dir=${3}/java_jni
+# The directory where the final binaries will be stored when scripts finish
+dist_dir=${4}
+prefix_dir="${build_dir}/java-jni"
+
+echo "=== Clear output directories and leftovers ==="
+# Clear output directories and leftovers
+rm -rf ${build_dir}
+
+echo "=== Building Arrow Java C Data Interface native library ==="
+mkdir -p "${build_dir}"
+pushd "${build_dir}"
+
+case "$(uname)" in
+ Linux)
+ n_jobs=$(nproc)
+ ;;
+ Darwin)
+ n_jobs=$(sysctl -n hw.logicalcpu)
+ ;;
+ *)
+ n_jobs=${NPROC:-1}
+ ;;
+esac
+
+: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}
+: ${CMAKE_BUILD_TYPE:=release}
+cmake \
+ -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \
+ -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \
+ -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \
+ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \
+ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \
+ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
+ -DProtobuf_USE_STATIC_LIBS=ON \
+ -GNinja \
+ ${JAVA_JNI_CMAKE_ARGS:-} \
+ ${arrow_dir}
+export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs}
+cmake --build . --config ${CMAKE_BUILD_TYPE}
+if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then
+ ctest \
+ --output-on-failure \
+ --parallel ${n_jobs} \
+ --timeout 300
+fi
+cmake --build . --config ${CMAKE_BUILD_TYPE} --target install
+popd
+
+mkdir -p ${dist_dir}
+# For Windows. *.dll are installed into bin/ on Windows.
+if [ -d "${prefix_dir}/bin" ]; then
+ mv ${prefix_dir}/bin/* ${dist_dir}/
+else
+ mv ${prefix_dir}/lib/* ${dist_dir}/
+fi
diff --git a/docker-compose.yml b/docker-compose.yml
index 103f2f3a..ae378865 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,18 +30,48 @@ volumes:
name: maven-cache
services:
- java:
+ ubuntu:
# Usage:
- # docker compose build java
- # docker compose run java
+ # docker compose build ubuntu
+ # docker compose run ubuntu
# Parameters:
- # MAVEN: 3.9.6
+ # MAVEN: 3.9.9
# JDK: 11, 17, 21
image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
- volumes: &java-volumes
+ volumes:
- .:/arrow-java:delegated
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
- command: &java-command >
+ command:
/bin/bash -c "
/arrow-java/ci/scripts/java_build.sh /arrow-java /build &&
/arrow-java/ci/scripts/java_test.sh /arrow-java /build"
+
+ conda-jni-cdata:
+ # Usage:
+ # docker compose build conda-jni-cdata
+ # docker compose run conda-jni-cdata
+ # Parameters:
+ # MAVEN: 3.9.9
+ # JDK: 11, 17, 21
+ image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration
+ build:
+ context: .
+ dockerfile: ci/docker/conda-jni.dockerfile
+ cache_from:
+ - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration
+ args:
+ jdk: ${JDK}
+ maven: ${MAVEN}
+ # required to use micromamba with rootless docker
+ #
https://github.com/mamba-org/micromamba-docker/issues/407#issuecomment-2088523507
+ user: root
+ volumes:
+ - .:/arrow-java:delegated
+ - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
+ environment:
+ ARROW_JAVA_CDATA: "ON"
+ command:
+ /bin/bash -c "
+ /arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build
/jni &&
+ /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni &&
+ /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni"
