This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3d585dac32 [CORE] Drop 15.0.0-gluten Arrow version rename and depend
on vanilla Apache Arrow (#12244)
3d585dac32 is described below
commit 3d585dac32750571020727066870d2205d93cfa7
Author: EJ Song <[email protected]>
AuthorDate: Tue Jun 9 13:44:01 2026 -0700
[CORE] Drop 15.0.0-gluten Arrow version rename and depend on vanilla Apache
Arrow (#12244)
---
.github/workflows/velox_backend_ansi.yml | 27 -------
.github/workflows/velox_backend_arm.yml | 12 ---
.github/workflows/velox_backend_enhanced.yml | 27 -------
.github/workflows/velox_backend_x86.yml | 108 ---------------------------
dev/build-arrow.sh | 17 ++++-
gluten-arrow/pom.xml | 10 +--
pom.xml | 3 -
7 files changed, 20 insertions(+), 184 deletions(-)
diff --git a/.github/workflows/velox_backend_ansi.yml
b/.github/workflows/velox_backend_ansi.yml
index 58e6825498..98b93a6f8d 100644
--- a/.github/workflows/velox_backend_ansi.yml
+++ b/.github/workflows/velox_backend_ansi.yml
@@ -124,8 +124,6 @@ jobs:
ccache -sz
bash dev/ci-velox-buildstatic-centos-7.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
"
- name: Save ccache
if: always()
@@ -138,11 +136,6 @@ jobs:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
spark-test-backends-velox-ansi41:
needs: build-native-lib
@@ -159,11 +152,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -236,11 +224,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -323,11 +306,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -400,11 +378,6 @@ jobs:
with:
name: velox-native-lib-ansi-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-ansi-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
diff --git a/.github/workflows/velox_backend_arm.yml
b/.github/workflows/velox_backend_arm.yml
index bbbe4fba6c..13a4515d08 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -68,8 +68,6 @@ jobs:
df -a
bash dev/ci-velox-buildstatic-centos-9.sh
ccache -s
- mkdir -p $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/*
$GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
- name: "Save ccache"
uses: actions/cache/save@v4
id: ccache
@@ -81,11 +79,6 @@ jobs:
name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-centos-8-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
tpc-test-centos8:
needs: build-native-lib-centos-8
@@ -104,11 +97,6 @@ jobs:
with:
name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-8-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
diff --git a/.github/workflows/velox_backend_enhanced.yml
b/.github/workflows/velox_backend_enhanced.yml
index ff669c7fa9..b3275df629 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -75,8 +75,6 @@ jobs:
mkdir -p /work/.ccache
bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
"
- name: "Save ccache"
@@ -90,11 +88,6 @@ jobs:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
spark-test-spark34:
needs: build-native-lib-centos-7
@@ -107,11 +100,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
run: |
dnf module -y install python39 && \
@@ -165,11 +153,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.5.5 (other tests)
run: |
dnf module -y install python39 && \
@@ -226,11 +209,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark35
@@ -277,11 +255,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-enhanced-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 159ed22b61..3b97c86b3a 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -79,8 +79,6 @@ jobs:
ccache -sz
bash dev/ci-velox-buildstatic-centos-7.sh
ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
"
- name: "Save ccache"
@@ -95,11 +93,6 @@ jobs:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
if-no-files-found: error
- - uses: actions/upload-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- if-no-files-found: error
tpc-test-ubuntu:
needs: build-native-lib-centos-7
@@ -157,11 +150,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Setup tzdata
run: |
sed -i 's|http://archive|http://eu.archive|g' /etc/apt/sources.list
@@ -184,7 +172,6 @@ jobs:
apt-get update && apt-get install -y openjdk-8-jdk wget
apt remove openjdk-11* -y
fi
- ls -l
/root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/
- name: Install Hadoop & Setup HDFS
if: matrix.os == 'ubuntu:22.04' && matrix.spark == 'spark-3.5' &&
matrix.java == 'java-8'
shell: bash
@@ -280,11 +267,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
if [ "${{ matrix.os }}" = "centos:7" ] || [ "${{ matrix.os }}" =
"centos:8" ]; then
@@ -347,11 +329,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: .m2/repository/org/apache/arrow/
- name: Build and run TPCH/DS tests
run: |
docker pull centos:7
@@ -406,11 +383,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /home/runner/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
sudo apt-get update
@@ -521,11 +493,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /home/runner/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
sudo apt-get update
@@ -564,11 +531,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Install Uniffle ${{ matrix.uniffle }}
run: |
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
@@ -615,11 +577,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
@@ -689,11 +646,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -749,11 +701,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.3.1 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark33
@@ -799,11 +746,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
run: |
dnf module -y install python39 && \
@@ -860,11 +802,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.4.4 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark34
@@ -913,11 +850,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -971,11 +903,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1025,11 +952,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark35
@@ -1076,11 +998,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1129,11 +1046,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark35
@@ -1295,11 +1207,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf module -y install python39 && \
@@ -1353,11 +1260,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark40
@@ -1404,11 +1306,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare
run: |
dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -1464,11 +1361,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image
update
run: |
rm -rf /opt/shims/spark41
diff --git a/dev/build-arrow.sh b/dev/build-arrow.sh
index 1bf4ef9d08..007047a8ad 100755
--- a/dev/build-arrow.sh
+++ b/dev/build-arrow.sh
@@ -75,6 +75,21 @@ function build_arrow_cpp() {
}
function build_arrow_java() {
+ # Maven Central's arrow-c-data / arrow-dataset jars at
${VELOX_ARROW_BUILD_VERSION}
+ # already ship libarrow_cdata_jni / libarrow_dataset_jni for x86_64
(Linux/macOS/Windows)
+ # and aarch_64 (Linux/macOS), so contributors on those archs do not need a
locally-built
+ # jar — gluten-arrow resolves the same artifact transitively.
+ #
+ # ppc64le has no native in the Central jar; support_ibm_power.patch
(applied above) adds
+ # the ppc64le -> ppcle_64 arch case to JniLoader.java and the local mvn
install step bakes
+ # a locally-built libarrow_cdata_jni.so for ppc64le into the resulting
arrow-c-data jar in
+ # ~/.m2, overriding Central. Skip the Java build on every other arch.
+ local ARCH=$(uname -m)
+ if [[ "${ARCH}" != "ppc64le" ]]; then
+ echo "Skipping local Arrow Java build on ${ARCH} — gluten resolves
arrow-c-data:${VELOX_ARROW_BUILD_VERSION} from Maven Central. Local build is
only required on ppc64le for the patched JniLoader."
+ return 0
+ fi
+
ARROW_INSTALL_DIR="${ARROW_PREFIX}/install"
# Use Gluten's Maven wrapper
@@ -96,8 +111,6 @@ function build_arrow_java() {
export CMAKE_BUILD_PARALLEL_LEVEL=$NPROC
pushd $ARROW_PREFIX/java
- # Because arrow-bom module need the -DprocessAllModules
- ${MVN_CMD} versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules
${MVN_CMD} clean install -pl
bom,maven/module-info-compiler-maven-plugin,vector -am \
-DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip
-Dassembly.skipAssembly
diff --git a/gluten-arrow/pom.xml b/gluten-arrow/pom.xml
index 62d9bd243a..defcfcbf26 100644
--- a/gluten-arrow/pom.xml
+++ b/gluten-arrow/pom.xml
@@ -88,13 +88,13 @@
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>${arrow-memory.artifact}</artifactId>
- <version>${arrow-gluten.version}</version>
+ <version>${arrow.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
- <version>${arrow-gluten.version}</version>
+ <version>${arrow.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
@@ -110,7 +110,7 @@
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
- <version>${arrow-gluten.version}</version>
+ <version>${arrow.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -129,7 +129,7 @@
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-c-data</artifactId>
- <version>${arrow-gluten.version}</version>
+ <version>${arrow.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
@@ -145,7 +145,7 @@
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-dataset</artifactId>
- <version>${arrow-gluten.version}</version>
+ <version>${arrow.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
diff --git a/pom.xml b/pom.xml
index 1b3a9d2a3c..fc4dbb273f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,6 @@
<celeborn.version>0.6.3</celeborn.version>
<uniffle.version>0.10.0</uniffle.version>
<arrow.version>15.0.0</arrow.version>
- <arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>
<arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
<hadoop.version>2.7.4</hadoop.version>
<antlr4.version>4.9.3</antlr4.version>
@@ -1287,7 +1286,6 @@
<log4j.version>2.24.3</log4j.version>
<commons-lang3.version>3.17.0</commons-lang3.version>
<arrow.version>18.1.0</arrow.version>
- <arrow-gluten.version>18.1.0</arrow-gluten.version>
<scala.compiler.version>4.9.2</scala.compiler.version>
</properties>
<dependencies>
@@ -1367,7 +1365,6 @@
<log4j.version>2.24.3</log4j.version>
<commons-lang3.version>3.17.0</commons-lang3.version>
<arrow.version>18.1.0</arrow.version>
- <arrow-gluten.version>18.1.0</arrow-gluten.version>
<scala.compiler.version>4.9.5</scala.compiler.version>
</properties>
<dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
