This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6972ecdc7d GH-44829: [Java][CI] Remove Java related test CI except
integration test (#44946)
6972ecdc7d is described below
commit 6972ecdc7dafaa99ffdfc0aa336479d1f621328e
Author: Sutou Kouhei <[email protected]>
AuthorDate: Thu Dec 5 11:41:46 2024 +0900
GH-44829: [Java][CI] Remove Java related test CI except integration test
(#44946)
### Rationale for this change
We don't need CI in apache/arrow except integration test because we're
moving the Java implementation to apache/arrow-java.
### What changes are included in this PR?
Remove Java related CI.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #44829
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.github/workflows/dev_pr/labeler.yml | 6 -
.github/workflows/java.yml | 173 -------------------
.github/workflows/java_jni.yml | 147 ----------------
.github/workflows/java_nightly.yml | 139 ----------------
ci/scripts/install_substrait_consumer.sh | 6 +-
ci/scripts/integration_spark.sh | 31 +---
dev/tasks/java-jars/README.md | 29 ----
dev/tasks/java-jars/github.yml | 278 -------------------------------
dev/tasks/tasks.yml | 180 +-------------------
docker-compose.yml | 123 +-------------
10 files changed, 8 insertions(+), 1104 deletions(-)
diff --git a/.github/workflows/dev_pr/labeler.yml
b/.github/workflows/dev_pr/labeler.yml
index fed4b77295..7ef92f0be9 100644
--- a/.github/workflows/dev_pr/labeler.yml
+++ b/.github/workflows/dev_pr/labeler.yml
@@ -35,11 +35,6 @@
- any-glob-to-any-file:
- go/**/*
-"Component: Java":
-- changed-files:
- - any-glob-to-any-file:
- - java/**/*
-
"Component: JavaScript":
- changed-files:
- any-glob-to-any-file:
@@ -82,7 +77,6 @@
- any-glob-to-any-file:
- c_glib/gandiva-glib/**/*
- cpp/src/gandiva/**/*
- - java/gandiva/**/*
- python/pyarrow/gandiva.*
- ruby/red-gandiva/**/*
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
deleted file mode 100644
index f6b3fa748e..0000000000
--- a/.github/workflows/java.yml
+++ /dev/null
@@ -1,173 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Java
-
-on:
- push:
- branches:
- - '**'
- - '!dependabot/**'
- tags:
- - '**'
- paths:
- - '.dockerignore'
- - '.github/workflows/java.yml'
- - 'ci/docker/*java*'
- - 'ci/scripts/java*.sh'
- - 'ci/scripts/util_*.sh'
- - 'docker-compose.yml'
- - 'format/Flight.proto'
- - 'java/**'
- pull_request:
- paths:
- - '.dockerignore'
- - '.github/workflows/java.yml'
- - 'ci/docker/*java*'
- - 'ci/scripts/java*.sh'
- - 'ci/scripts/util_*.sh'
- - 'docker-compose.yml'
- - 'format/Flight.proto'
- - 'java/**'
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-env:
- ARCHERY_DEBUG: 1
- DOCKER_VOLUME_PREFIX: ".docker/"
-
-jobs:
- ubuntu:
- name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven
}}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 30
- strategy:
- fail-fast: false
- matrix:
- jdk: [11, 17, 21, 22]
- maven: [3.9.6]
- image: [java]
- env:
- JDK: ${{ matrix.jdk }}
- MAVEN: ${{ matrix.maven }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Cache Docker Volumes
- uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
- with:
- path: .docker
- key: maven-${{ hashFiles('java/**') }}
- restore-keys: maven-
- - name: Setup Python
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #
v5.3.0
- with:
- python-version: 3.12
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: |
- archery docker run \
- -e CI=true \
- -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
- ${{ matrix.image }}
- - name: Docker Push
- if: >-
- success() &&
- github.event_name == 'push' &&
- github.repository == 'apache/arrow' &&
- github.ref_name == 'main'
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
-
- macos:
- name: AMD64 macOS 13 Java JDK ${{ matrix.jdk }}
- runs-on: macos-13
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 30
- strategy:
- fail-fast: false
- matrix:
- jdk: [11]
- steps:
- - name: Set up Java
- uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: ${{ matrix.jdk }}
- - name: Checkout Arrow
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Build
- shell: bash
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
-
- windows:
- name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }}
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 30
- strategy:
- fail-fast: false
- matrix:
- jdk: [11]
- steps:
- - name: Set up Java
- uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.jdk }}
- distribution: 'temurin'
- - name: Checkout Arrow
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Build
- shell: bash
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
deleted file mode 100644
index 5682f8e841..0000000000
--- a/.github/workflows/java_jni.yml
+++ /dev/null
@@ -1,147 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Java JNI
-
-on:
- push:
- branches:
- - '**'
- - '!dependabot/**'
- tags:
- - '**'
- paths:
- - '.dockerignore'
- - '.github/workflows/java_jni.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_build.sh'
- - 'ci/scripts/java_*'
- - 'cpp/**'
- - 'docker-compose.yml'
- - 'java/**'
- pull_request:
- paths:
- - '.dockerignore'
- - '.github/workflows/java_jni.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_build.sh'
- - 'ci/scripts/java_*'
- - 'cpp/**'
- - 'docker-compose.yml'
- - 'java/**'
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-env:
- ARCHERY_DEBUG: 1
- DOCKER_VOLUME_PREFIX: ".docker/"
-
-jobs:
- docker:
- name: AMD64 manylinux2014 Java JNI
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 240
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Free up disk space
- run: |
- ci/scripts/util_free_space.sh
- - name: Cache Docker Volumes
- uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
- with:
- path: .docker
- key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }}
- restore-keys: java-jni-manylinux-2014-
- - name: Setup Python
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #
v5.3.0
- with:
- python-version: 3.12
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- run: |
- source ci/scripts/util_enable_core_dumps.sh
- archery docker run java-jni-manylinux-2014
- - name: Docker Push
- if: >-
- success() &&
- github.event_name == 'push' &&
- github.repository == 'apache/arrow' &&
- github.ref_name == 'main'
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- continue-on-error: true
- run: archery docker push java-jni-manylinux-2014
-
- docker_integration_python:
- name: AMD64 Conda Java C Data Interface Integration
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- timeout-minutes: 90
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
- with:
- fetch-depth: 0
- submodules: recursive
- - name: Cache Docker Volumes
- uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
- with:
- path: .docker
- key: maven-${{ hashFiles('java/**') }}
- restore-keys: maven-
- - name: Setup Python
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #
v5.3.0
- with:
- python-version: 3.12
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- run: |
- archery docker run \
- -e CI=true \
- -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
- conda-python-java-integration
- - name: Docker Push
- if: >-
- success() &&
- github.event_name == 'push' &&
- github.repository == 'apache/arrow' &&
- github.ref_name == 'main'
- env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
- continue-on-error: true
- run: archery docker push conda-python-java-integration
diff --git a/.github/workflows/java_nightly.yml
b/.github/workflows/java_nightly.yml
deleted file mode 100644
index 436cc324dd..0000000000
--- a/.github/workflows/java_nightly.yml
+++ /dev/null
@@ -1,139 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Upload Java Nightly builds
-
-on:
- workflow_dispatch:
- inputs:
- prefix:
- description: Job prefix to use.
- required: false
- default: ''
- keep:
- description: Number of versions to keep.
- required: false
- default: 14
- schedule:
- - cron: '0 14 * * *'
-
-permissions:
- contents: read
-
-jobs:
- upload:
- if: github.repository == 'apache/arrow'
- env:
- PREFIX: ${{ github.event.inputs.prefix || ''}}
- CROSSBOW_GITHUB_TOKEN: ${{ github.token }}
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
- with:
- fetch-depth: 1
- path: arrow
- repository: apache/arrow
- ref: main
- submodules: recursive
- - name: Checkout Crossbow
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
- with:
- fetch-depth: 0
- path: crossbow
- repository: ursacomputing/crossbow
- ref: main
- - name: Set up Python
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #
v5.3.0
- with:
- cache: 'pip'
- python-version: 3.12
- - name: Install Archery
- shell: bash
- run: pip install -e arrow/dev/archery[all]
- - run: mkdir -p binaries
- - name: Download Artifacts
- run: |
- if [ -z $PREFIX ]; then
- PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
- fi
- echo $PREFIX
- archery crossbow download-artifacts -f java-jars -t binaries $PREFIX
- - name: Sync from Remote
- uses: ./arrow/.github/actions/sync-nightlies
- with:
- switches: -avzh --update --delete --progress
- local_path: repo
- remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
- remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
- remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
- remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
- remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
- remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
- - shell: bash
- name: Show local repo sync from remote
- run: |
- for i in `ls -t repo/org/apache/arrow`; do
- echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth
1 -type d \
- | wc -l \
- | xargs) versions available"
- done
- - shell: bash
- name: Build Repository
- run: |
- DATE=$(date +%Y-%m-%d)
- if [ -z $PREFIX ]; then
- PREFIX=nightly-packaging-${DATE}-0
- fi
-
PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)'
- mkdir -p repo/org/apache/arrow/
- for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E
'.jar|.json|.pom|.xml' | grep SNAPSHOT); do
- [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]]
- mkdir -p
repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
- mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
- # Copy twice to maintain a latest snapshot and some earlier
versions
- cp binaries/$PREFIX/java-jars/$LIBRARY
repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
- touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
- cp binaries/$PREFIX/java-jars/$LIBRARY
repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
- echo "Artifacts $LIBRARY configured"
- done
- - name: Prune Repository
- shell: bash
- env:
- KEEP: ${{ github.event.inputs.keep || 14 }}
- run: |
- for i in `ls -t repo/org/apache/arrow`; do
- find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d
-print0 \
- | xargs -0 ls -t -d \
- | tail -n +$((KEEP + 1)) \
- | xargs rm -rf
- done
- - name: Show repo contents
- run: tree repo
- - name: Sync to Remote
- if: ${{ github.repository == 'apache/arrow' }}
- uses: ./arrow/.github/actions/sync-nightlies
- with:
- upload: true
- switches: -avzh --update --delete --progress
- local_path: repo
- remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
- remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
- remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
- remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
- remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
- remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
diff --git a/ci/scripts/install_substrait_consumer.sh
b/ci/scripts/install_substrait_consumer.sh
index 8d3333a18f..2e6d299f68 100755
--- a/ci/scripts/install_substrait_consumer.sh
+++ b/ci/scripts/install_substrait_consumer.sh
@@ -24,10 +24,8 @@ echo "Install Substrait Consumer Test Suite";
git clone https://github.com/substrait-io/consumer-testing.git
cd consumer-testing
# avoid installing pyarrow
-grep -v 'pyarrow\|arrow-nightlies' requirements.txt | while read line
-do
- pip install $line
-done
+grep -v 'pyarrow\|arrow-nightlies' requirements.txt > requirements-no-arrow.txt
+pip install -r requirements-no-arrow.txt
pip install -r requirements-build.txt
# setup substrait-java
diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh
index 424ac59946..4272e2e776 100755
--- a/ci/scripts/integration_spark.sh
+++ b/ci/scripts/integration_spark.sh
@@ -21,9 +21,6 @@ set -eu
source_dir=${1}
spark_dir=${2}
-# Test Spark with latest PyArrow only, don't build with latest Arrow Java
-test_pyarrow_only=${3:-false}
-
# Spark branch to checkout
spark_version=${SPARK_VERSION:-master}
@@ -44,32 +41,10 @@ export MAVEN_OPTS="-Xss256m -Xmx2g
-XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simp
export MAVEN_OPTS="${MAVEN_OPTS}
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
pushd ${spark_dir}
+ echo "Building Spark ${SPARK_VERSION}"
- if [ "${test_pyarrow_only}" == "true" ]; then
- echo "Building Spark ${SPARK_VERSION} to test pyarrow only"
-
- # Build Spark only
- build/mvn -B -DskipTests package
-
- else
-
- # Update Spark pom with the Arrow version just installed and build Spark,
need package phase for pyspark
- echo "Building Spark ${SPARK_VERSION} with Arrow ${arrow_version}"
- build/mvn versions:set-property -Dproperty=arrow.version
-DnewVersion=${arrow_version}
-
- # Build Spark with new Arrow Java
- build/mvn -B -DskipTests package
-
- spark_scala_tests=(
- "org.apache.spark.sql.execution.arrow"
- "org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite"
- "org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite")
-
- (echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}")
-
- # TODO: should be able to only build spark-sql tests with adding "-pl
sql/core" but not currently working
- build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo
"${spark_scala_tests[*]}") test
- fi
+ # Build Spark only
+ build/mvn -B -DskipTests package
# Run pyarrow related Python tests only
spark_python_tests=(
diff --git a/dev/tasks/java-jars/README.md b/dev/tasks/java-jars/README.md
deleted file mode 100644
index 216c7198d3..0000000000
--- a/dev/tasks/java-jars/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Java Jars Task
-
-This directory is responsible to generate the jar files for the Arrow
components that depend on C++ shared libraries to execute.
-
-The Arrow C++ libraries are compiled both on macOS and Linux distributions,
with their dependencies linked statically, and they are added
-in the jars at the end, so the file can be used on both systems.
-
-## Linux Docker Image
-To compile the C++ libraries in Linux, a docker image is used.
-It is created used the **ci/docker/java-bundled-jars.dockerfile** file.
-If it is necessary to add any new dependency, you need to change that file.
\ No newline at end of file
diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
deleted file mode 100644
index affa26a1d9..0000000000
--- a/dev/tasks/java-jars/github.yml
+++ /dev/null
@@ -1,278 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-{% import 'macros.jinja' as macros with context %}
-
-{{ macros.github_header() }}
-
-permissions:
- packages: write
-
-jobs:
-
- build-cpp-ubuntu:
- {% set arch = '${{ matrix.platform.arch }}' %}
- name: Build C++ libraries Ubuntu {{ arch }}
- runs-on: {{ '${{ matrix.platform.runs_on }}' }}
- env:
- # architecture name used for archery build
- ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
- ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
- ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
- strategy:
- fail-fast: false
- matrix:
- platform:
- - runs_on: ["ubuntu-latest"]
- arch: "x86_64"
- archery_arch: "amd64"
- archery_arch_alias: "x86_64"
- archery_arch_short: "amd64"
- - runs_on: ["self-hosted", "Linux", "arm64"]
- arch: "aarch_64"
- archery_arch: "arm64v8"
- archery_arch_alias: "aarch64"
- archery_arch_short: "arm64"
- steps:
- {{ macros.github_checkout_arrow()|indent }}
- {{ macros.github_free_space()|indent }}
- {{ macros.github_install_archery()|indent }}
- - name: Build C++ libraries
- env:
- {{ macros.github_set_sccache_envvars()|indent(8) }}
- GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }}
- run: |
- if [ "${ARCH}" = "arm64v8" ]; then
- # We can't use NuGet on manylinux2014_aarch64 because Mono is old.
- :
- else
- export VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite"
- fi
- archery docker run \
- -e ARROW_JAVA_BUILD=OFF \
- -e ARROW_JAVA_TEST=OFF \
- java-jni-manylinux-2014
- - name: Compress into single artifact to keep directory structure
- run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz
arrow/java-dist/
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: ubuntu-shared-lib-{{ arch }}
- path: arrow-shared-libs-linux-{{ arch }}.tar.gz
- {% if arrow.is_default_branch() %}
- {{ macros.github_login_dockerhub()|indent }}
- - name: Push Docker image
- shell: bash
- run: archery docker push java-jni-manylinux-2014
- {% endif %}
-
- build-cpp-macos:
- {% set arch = '${{ matrix.platform.arch }}' %}
- name: Build C++ libraries macOS {{ arch }}
- runs-on: {{ '${{ matrix.platform.runs_on }}' }}
- strategy:
- fail-fast: false
- matrix:
- platform:
- - { runs_on: ["macos-13"], arch: "x86_64"}
- - { runs_on: ["macos-14"], arch: "aarch_64" }
- env:
- MACOSX_DEPLOYMENT_TARGET: "14.0"
- steps:
- {{ macros.github_checkout_arrow()|indent }}
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- cache: 'pip'
- python-version: 3.12
- - name: Install Archery
- shell: bash
- run: pip install -e arrow/dev/archery[all]
- - name: Install dependencies
- run: |
- # We want to use llvm@14 to avoid shared z3
- # dependency. llvm@14 doesn't depend on z3 and llvm depends
- # on z3. And Homebrew's z3 provides only shared library. It
- # doesn't provides static z3 because z3's CMake doesn't accept
- # building both shared and static libraries at once.
- # See also: Z3_BUILD_LIBZ3_SHARED in
- # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
- #
- # If llvm is installed, Apache Arrow C++ uses llvm rather than
- # llvm@14 because llvm is newer than llvm@14.
- brew uninstall llvm || :
-
- # Ensure updating python@XXX with the "--overwrite" option.
- # If python@XXX is updated without "--overwrite", it causes
- # a conflict error. Because Python 3 installed not by
- # Homebrew exists in /usr/local on GitHub Actions. If
- # Homebrew's python@XXX is updated without "--overwrite", it
- # tries to replace /usr/local/bin/2to3 and so on and causes
- # a conflict error.
- brew update
- for python_package in $(brew list | grep python@); do
- brew install --overwrite ${python_package}
- done
- brew install --overwrite python
-
- if [ "$(uname -m)" = "arm64" ]; then
- # pkg-config formula is deprecated but it's still installed
- # in GitHub Actions runner now. We can remove this once
- # pkg-config formula is removed from GitHub Actions runner.
- brew uninstall pkg-config || :
- brew uninstall [email protected] || :
- fi
-
- brew bundle --file=arrow/cpp/Brewfile
- # We want to link aws-sdk-cpp statically but Homebrew's
- # aws-sdk-cpp provides only shared library. If we have
- # Homebrew's aws-sdk-cpp, our build mix Homebrew's
- # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
- # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
- brew uninstall aws-sdk-cpp
- # We want to use bundled RE2 for static linking. If
- # Homebrew's RE2 is installed, its header file may be used.
- # We uninstall Homebrew's RE2 to ensure using bundled RE2.
- brew uninstall grpc || : # gRPC depends on RE2
- brew uninstall [email protected] || : # gRPC 1.54 may be installed too
- brew uninstall re2
- # We want to use bundled Protobuf for static linking. If
- # Homebrew's Protobuf is installed, its library file may be
- # used on test We uninstall Homebrew's Protobuf to ensure using
- # bundled Protobuf.
- brew uninstall protobuf
-
- brew bundle --file=arrow/java/Brewfile
- - name: Build C++ libraries
- env:
- {{ macros.github_set_sccache_envvars()|indent(8) }}
- run: |
- set -e
- # make brew Java available to CMake
- export JAVA_HOME=$(brew --prefix
openjdk@11)/libexec/openjdk.jdk/Contents/Home
- arrow/ci/scripts/java_jni_macos_build.sh \
- $GITHUB_WORKSPACE/arrow \
- $GITHUB_WORKSPACE/arrow/cpp-build \
- $GITHUB_WORKSPACE/arrow/java-dist
- - name: Compress into single artifact to keep directory structure
- run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz
arrow/java-dist/
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: macos-shared-lib-{{ arch }}
- path: arrow-shared-libs-macos-{{ arch }}.tar.gz
-
- build-cpp-windows:
- name: Build C++ libraries Windows
- runs-on: windows-2019
- steps:
- {{ macros.github_checkout_arrow()|indent }}
- - name: Set up Java
- uses: actions/setup-java@v3
- with:
- java-version: '11'
- distribution: 'temurin'
- - name: Download Timezone Database
- shell: bash
- run: arrow/ci/scripts/download_tz_database.sh
- - name: Install sccache
- shell: bash
- run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
- - name: Build C++ libraries
- shell: cmd
- env:
- {{ macros.github_set_sccache_envvars()|indent(8) }}
- run: |
- call "C:\Program Files (x86)\Microsoft Visual
Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
- REM For ORC
- set TZDIR=/c/msys64/usr/share/zoneinfo
- bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow
$(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
- - name: Compress into single artifact to keep directory structure
- shell: bash
- run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: windows-shared-lib
- path: arrow-shared-libs-windows.tar.gz
-
- package-jars:
- name: Build jar files
- runs-on: macos-latest
- needs:
- - build-cpp-ubuntu
- - build-cpp-macos
- - build-cpp-windows
- steps:
- {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
- - name: Download Libraries
- uses: actions/download-artifact@v4
- with:
- path: artifacts
- - name: Decompress artifacts
- run: |
- mv artifacts/*/*.tar.gz .
- tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
- tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
- tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
- tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
- tar -xvzf arrow-shared-libs-windows.tar.gz
- - name: Test that shared libraries exist
- run: |
- set -x
-
- test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
- test -f
arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
- test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
- test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so
-
- test -f
arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
- test -f
arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
- test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
- test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so
-
- test -f
arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
- test -f
arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
- test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
- test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib
-
- test -f
arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
- test -f
arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
- test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
- test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
-
- test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
- test -f
arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
- test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
- - name: Build bundled jar
- env:
- MAVEN_ARGS: >-
- --no-transfer-progress
- run: |
- set -e
- pushd arrow/java
- mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }}
- mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f
bom
- popd
- arrow/ci/scripts/java_full_build.sh \
- $GITHUB_WORKSPACE/arrow \
- $GITHUB_WORKSPACE/arrow/java-dist
- {{ macros.github_upload_releases(["arrow/java-dist/*.jar",
- "arrow/java-dist/*.json",
- "arrow/java-dist/*.pom",
- "arrow/java-dist/*.xml",
- "arrow/java-dist/*.zip"])|indent }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 69235bf0cf..c8c311f513 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -59,7 +59,6 @@ groups:
- centos-*
- conan-*
- debian-*
- - java-jars
- matlab
- nuget
- python-sdist
@@ -83,10 +82,6 @@ groups:
c-glib:
- test-*c-glib*
- java:
- - "*java*"
- - test-*spark*
-
python:
- test-*python*
- example-*python*
@@ -102,7 +97,6 @@ groups:
vcpkg:
- test-*vcpkg*
- wheel-*
- - java-jars
integration:
- test-*dask*
@@ -154,7 +148,6 @@ groups:
- centos-*
- conan-*
- conda-*
- - java-jars
- homebrew-cpp
- nuget
- test-*
@@ -184,7 +177,6 @@ groups:
- ~conda-linux-x64-cuda-py3
- ~conda-osx-arm64-cpu-py3
- conan-*
- - java-jars
- homebrew-cpp
- nuget
- wheel-*
@@ -695,173 +687,6 @@ tasks:
artifacts:
- matlab-arrow-{no_rc_no_dev_version}.mltbx
- ############################## Arrow JAR's ##################################
-
- java-jars:
- # Build jar's that contains cpp libraries dependencies
- ci: github
- template: java-jars/github.yml
- artifacts:
- - arrow-algorithm-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-algorithm-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-algorithm-{no_rc_snapshot_version}-javadoc.jar
- - arrow-algorithm-{no_rc_snapshot_version}-sources.jar
- - arrow-algorithm-{no_rc_snapshot_version}-tests.jar
- - arrow-algorithm-{no_rc_snapshot_version}.jar
- - arrow-algorithm-{no_rc_snapshot_version}.pom
- - arrow-avro-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-avro-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-avro-{no_rc_snapshot_version}-javadoc.jar
- - arrow-avro-{no_rc_snapshot_version}-sources.jar
- - arrow-avro-{no_rc_snapshot_version}-tests.jar
- - arrow-avro-{no_rc_snapshot_version}.jar
- - arrow-avro-{no_rc_snapshot_version}.pom
- - arrow-bom-{no_rc_snapshot_version}.pom
- - arrow-c-data-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-c-data-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-c-data-{no_rc_snapshot_version}-javadoc.jar
- - arrow-c-data-{no_rc_snapshot_version}-sources.jar
- - arrow-c-data-{no_rc_snapshot_version}-tests.jar
- - arrow-c-data-{no_rc_snapshot_version}.jar
- - arrow-c-data-{no_rc_snapshot_version}.pom
- - arrow-compression-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-compression-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-compression-{no_rc_snapshot_version}-javadoc.jar
- - arrow-compression-{no_rc_snapshot_version}-sources.jar
- - arrow-compression-{no_rc_snapshot_version}-tests.jar
- - arrow-compression-{no_rc_snapshot_version}.jar
- - arrow-compression-{no_rc_snapshot_version}.pom
- - arrow-dataset-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-dataset-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-dataset-{no_rc_snapshot_version}-javadoc.jar
- - arrow-dataset-{no_rc_snapshot_version}-sources.jar
- - arrow-dataset-{no_rc_snapshot_version}-tests.jar
- - arrow-dataset-{no_rc_snapshot_version}.jar
- - arrow-dataset-{no_rc_snapshot_version}.pom
- - arrow-flight-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-flight-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-flight-{no_rc_snapshot_version}.pom
- - arrow-format-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-format-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-format-{no_rc_snapshot_version}-javadoc.jar
- - arrow-format-{no_rc_snapshot_version}-sources.jar
- - arrow-format-{no_rc_snapshot_version}-tests.jar
- - arrow-format-{no_rc_snapshot_version}.jar
- - arrow-format-{no_rc_snapshot_version}.pom
- - arrow-gandiva-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-gandiva-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-gandiva-{no_rc_snapshot_version}-javadoc.jar
- - arrow-gandiva-{no_rc_snapshot_version}-sources.jar
- - arrow-gandiva-{no_rc_snapshot_version}-tests.jar
- - arrow-gandiva-{no_rc_snapshot_version}.jar
- - arrow-gandiva-{no_rc_snapshot_version}.pom
- - arrow-java-root-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-java-root-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-java-root-{no_rc_snapshot_version}-source-release.zip
- - arrow-java-root-{no_rc_snapshot_version}.pom
- - arrow-jdbc-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-jdbc-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-jdbc-{no_rc_snapshot_version}-javadoc.jar
- - arrow-jdbc-{no_rc_snapshot_version}-sources.jar
- - arrow-jdbc-{no_rc_snapshot_version}-tests.jar
- - arrow-jdbc-{no_rc_snapshot_version}.jar
- - arrow-jdbc-{no_rc_snapshot_version}.pom
- - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-memory-core-{no_rc_snapshot_version}-javadoc.jar
- - arrow-memory-core-{no_rc_snapshot_version}-sources.jar
- - arrow-memory-core-{no_rc_snapshot_version}-tests.jar
- - arrow-memory-core-{no_rc_snapshot_version}.jar
- - arrow-memory-core-{no_rc_snapshot_version}.pom
- - arrow-memory-netty-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-memory-netty-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-memory-netty-{no_rc_snapshot_version}-javadoc.jar
- - arrow-memory-netty-{no_rc_snapshot_version}-sources.jar
- - arrow-memory-netty-{no_rc_snapshot_version}-tests.jar
- - arrow-memory-netty-{no_rc_snapshot_version}.jar
- - arrow-memory-netty-{no_rc_snapshot_version}.pom
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-javadoc.jar
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-sources.jar
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-tests.jar
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}.jar
- - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}.pom
- - arrow-memory-unsafe-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-memory-unsafe-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-memory-unsafe-{no_rc_snapshot_version}-javadoc.jar
- - arrow-memory-unsafe-{no_rc_snapshot_version}-sources.jar
- - arrow-memory-unsafe-{no_rc_snapshot_version}-tests.jar
- - arrow-memory-unsafe-{no_rc_snapshot_version}.jar
- - arrow-memory-unsafe-{no_rc_snapshot_version}.pom
- - arrow-memory-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-memory-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-memory-{no_rc_snapshot_version}.pom
- - arrow-orc-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-orc-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-orc-{no_rc_snapshot_version}-javadoc.jar
- - arrow-orc-{no_rc_snapshot_version}-sources.jar
- - arrow-orc-{no_rc_snapshot_version}-tests.jar
- - arrow-orc-{no_rc_snapshot_version}.jar
- - arrow-orc-{no_rc_snapshot_version}.pom
- - arrow-performance-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-performance-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-performance-{no_rc_snapshot_version}-sources.jar
- - arrow-performance-{no_rc_snapshot_version}-tests.jar
- - arrow-performance-{no_rc_snapshot_version}.jar
- - arrow-performance-{no_rc_snapshot_version}.pom
- - arrow-tools-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-tools-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-tools-{no_rc_snapshot_version}-jar-with-dependencies.jar
- - arrow-tools-{no_rc_snapshot_version}-javadoc.jar
- - arrow-tools-{no_rc_snapshot_version}-sources.jar
- - arrow-tools-{no_rc_snapshot_version}-tests.jar
- - arrow-tools-{no_rc_snapshot_version}.jar
- - arrow-tools-{no_rc_snapshot_version}.pom
- - arrow-vector-{no_rc_snapshot_version}-cyclonedx.json
- - arrow-vector-{no_rc_snapshot_version}-cyclonedx.xml
- - arrow-vector-{no_rc_snapshot_version}-javadoc.jar
- - arrow-vector-{no_rc_snapshot_version}-shade-format-flatbuffers.jar
- - arrow-vector-{no_rc_snapshot_version}-sources.jar
- - arrow-vector-{no_rc_snapshot_version}-tests.jar
- - arrow-vector-{no_rc_snapshot_version}.jar
- - arrow-vector-{no_rc_snapshot_version}.pom
- - flight-core-{no_rc_snapshot_version}-cyclonedx.json
- - flight-core-{no_rc_snapshot_version}-cyclonedx.xml
- - flight-core-{no_rc_snapshot_version}-javadoc.jar
- - flight-core-{no_rc_snapshot_version}-sources.jar
- - flight-core-{no_rc_snapshot_version}-tests.jar
- - flight-core-{no_rc_snapshot_version}.jar
- - flight-core-{no_rc_snapshot_version}.pom
- - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.json
- - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.xml
- -
flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar
- - flight-integration-tests-{no_rc_snapshot_version}-javadoc.jar
- - flight-integration-tests-{no_rc_snapshot_version}-sources.jar
- - flight-integration-tests-{no_rc_snapshot_version}-tests.jar
- - flight-integration-tests-{no_rc_snapshot_version}.jar
- - flight-integration-tests-{no_rc_snapshot_version}.pom
- - flight-sql-{no_rc_snapshot_version}-cyclonedx.json
- - flight-sql-{no_rc_snapshot_version}-cyclonedx.xml
- - flight-sql-{no_rc_snapshot_version}-javadoc.jar
- - flight-sql-{no_rc_snapshot_version}-sources.jar
- - flight-sql-{no_rc_snapshot_version}-tests.jar
- - flight-sql-{no_rc_snapshot_version}.jar
- - flight-sql-{no_rc_snapshot_version}.pom
- - flight-sql-jdbc-core-{no_rc_snapshot_version}-cyclonedx.json
- - flight-sql-jdbc-core-{no_rc_snapshot_version}-cyclonedx.xml
- - flight-sql-jdbc-core-{no_rc_snapshot_version}-javadoc.jar
- - flight-sql-jdbc-core-{no_rc_snapshot_version}-sources.jar
- - flight-sql-jdbc-core-{no_rc_snapshot_version}-tests.jar
- - flight-sql-jdbc-core-{no_rc_snapshot_version}.jar
- - flight-sql-jdbc-core-{no_rc_snapshot_version}.pom
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}-cyclonedx.json
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}-cyclonedx.xml
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}-sources.jar
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}-tests.jar
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}.jar
- - flight-sql-jdbc-driver-{no_rc_snapshot_version}.pom
-
############################## NuGet packages ###############################
nuget:
@@ -928,7 +753,6 @@ tasks:
{% for target in ["cpp",
"csharp",
"integration",
- "java",
"js",
"python",
"ruby"] %}
@@ -988,7 +812,6 @@ tasks:
{% for target in ["cpp",
"csharp",
"integration",
- "java",
"js",
"python",
"ruby"] %}
@@ -1013,7 +836,6 @@ tasks:
env:
ARROW_FLIGHT: 0
ARROW_GANDIVA: 0
- TEST_INTEGRATION_JAVA: 0
PYTEST_ADDOPTS: "-k 'not test_cancellation'"
target: {{ target }}
github_runner: "macos-14"
@@ -1354,7 +1176,7 @@ tasks:
test-r-extra-packages:
ci: github
- template: r/github.linux.extra.packages.yml
+ template: r/github.linux.extra.packages.yml
test-r-linux-as-cran:
ci: github
diff --git a/docker-compose.yml b/docker-compose.yml
index 6ee1bf4d11..c2b6ae23f5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -127,8 +127,6 @@ x-hierarchy:
- conda-python-dask
- conda-python-emscripten
- conda-python-hdfs
- - conda-python-java-integration
- - conda-python-jpype
- conda-python-no-numpy
- conda-python-spark
- conda-python-substrait
@@ -142,7 +140,6 @@ x-hierarchy:
- debian-js
- fedora-cpp:
- fedora-python
- - java
- python-sdist
- ubuntu-cpp:
- ubuntu-cpp-static
@@ -173,8 +170,7 @@ x-hierarchy:
# helper services
- impala
- postgres
- - python-wheel-manylinux-2014:
- - java-jni-manylinux-2014
+ - python-wheel-manylinux-2014
- python-wheel-manylinux-2-28
- python-wheel-manylinux-test-imports
- python-free-threaded-wheel-manylinux-test-imports
@@ -1301,28 +1297,6 @@ services:
target: "C:/arrow"
command: arrow\\ci\\scripts\\python_wheel_windows_test.bat
- java-jni-manylinux-2014:
- image: ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
- build:
- args:
- base:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
- java: 11
- context: .
- dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile
- cache_from:
- - ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
- environment:
- <<: [*common, *ccache]
- volumes:
- - .:/arrow:delegated
- - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
- -
${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
- command:
- ["pip install -e /arrow/dev/archery && \
- /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build
/arrow/java-dist && \
- /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \
- /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]
-
############################## Integration #################################
conda-python-pandas:
@@ -1478,79 +1452,6 @@ services:
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/integration_substrait.sh"]
- conda-python-jpype:
- # Usage:
- # docker compose build conda
- # docker compose build conda-cpp
- # docker compose build conda-python
- # docker compose build conda-python-jpype
- # docker compose run --rm conda-python-jpype
- image: ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
- build:
- context: .
- dockerfile: ci/docker/conda-python-jpype.dockerfile
- cache_from:
- - ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
- args:
- repo: ${REPO}
- arch: ${ARCH}
- python: ${PYTHON}
- shm_size: *shm-size
- environment:
- <<: [*common, *ccache]
- ARROW_FLIGHT: "OFF"
- ARROW_FLIGHT_SQL: "OFF"
- ARROW_GANDIVA: "OFF"
- volumes: *conda-volumes
- command:
- ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
- /arrow/ci/scripts/python_build.sh /arrow /build &&
- /arrow/ci/scripts/java_build.sh /arrow /build &&
- /arrow/ci/scripts/python_test.sh /arrow"]
-
- conda-python-java-integration:
- # Usage:
- # docker compose build conda
- # docker compose build conda-cpp
- # docker compose build conda-python
- # docker compose build conda-python-java-integration
- # docker compose run --rm conda-python-java-integration
- image: ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration
- build:
- context: .
- dockerfile: ci/docker/conda-python-jpype.dockerfile
- cache_from:
- - ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration
- args:
- repo: ${REPO}
- arch: ${ARCH}
- python: ${PYTHON}
- llvm: ${LLVM}
- shm_size: *shm-size
- environment:
- <<: [*common, *ccache]
- ARROW_ACERO: "OFF"
- ARROW_DATASET: "OFF"
- ARROW_FLIGHT: "OFF"
- ARROW_FLIGHT_SQL: "OFF"
- ARROW_GANDIVA: "OFF"
- ARROW_JAVA_CDATA: "ON"
- ARROW_ORC: "OFF"
- ARROW_PARQUET: "OFF"
- JAVA_JNI_CMAKE_ARGS: >-
- -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF
- -DARROW_JAVA_JNI_ENABLE_C=ON
- volumes:
- - .:/arrow:delegated
- - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
- - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
- command:
- [ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
- /arrow/ci/scripts/python_build.sh /arrow /build &&
- /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build
/tmp/dist/java/ &&
- /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
- /arrow/ci/scripts/java_cdata_integration.sh /arrow /build" ]
-
conda-python-cython2:
# Usage:
# docker compose build conda
@@ -1836,25 +1737,6 @@ services:
/arrow/ci/scripts/csharp_test.sh /arrow &&
/arrow/ci/scripts/csharp_pack.sh /arrow"
- ################################ Java #######################################
-
- java:
- # Usage:
- # docker compose build java
- # docker compose run java
- # Parameters:
- # MAVEN: 3.9.5
- # JDK: 11, 17, 21
- image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
- shm_size: *shm-size
- volumes: &java-volumes
- - .:/arrow:delegated
- - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
- command: &java-command >
- /bin/bash -c "
- /arrow/ci/scripts/java_build.sh /arrow /build &&
- /arrow/ci/scripts/java_test.sh /arrow /build"
-
############################## Integration ##################################
conda-integration:
@@ -2070,8 +1952,7 @@ services:
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
- /arrow/ci/scripts/java_build.sh /arrow /build &&
- /arrow/ci/scripts/integration_spark.sh /arrow /spark
${TEST_PYARROW_ONLY:-false}"]
+ /arrow/ci/scripts/integration_spark.sh /arrow /spark"]
################################# Source Verification
#####################################
