This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a35a3bda13 Add a project-wide mvn build script (#11322)
a35a3bda13 is described below
commit a35a3bda13b17e6cf51dd1f2d2e89b7fb3ecae20
Author: Kent Yao <[email protected]>
AuthorDate: Wed Dec 24 10:11:04 2025 +0800
Add a project-wide mvn build script (#11322)
* Add a project-wide mvn build script
* typo
* typo
---
.gitignore | 1 +
build/mvn | 159 +++++++++++++++++++++++++++++++++++++++++++++
dev/build-arrow.sh | 13 ++--
dev/buildbundle-veloxbe.sh | 17 +++--
dev/format-scala-code.sh | 8 ++-
dev/gen-all-config-docs.sh | 6 +-
pom.xml | 2 +
7 files changed, 191 insertions(+), 15 deletions(-)
diff --git a/.gitignore b/.gitignore
index ab4f7c54a5..899da20811 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ CMakeFiles/
CMakeCache.txt
CTestTestfile.cmake
cmake_install.cmake
+!build/mvn
build/
*-build/
Testing/
diff --git a/build/mvn b/build/mvn
new file mode 100755
index 0000000000..2c6e27ca76
--- /dev/null
+++ b/build/mvn
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script downloads and installs Apache Maven if it's not already installed
+# Usage: build/mvn <maven-args>
+#
+
+# Determine the current working directory
+GLUTEN_HOME="$(cd "$(dirname "$0")"/.. && pwd)" || exit 1
+DOWNLOAD_DIR="${GLUTEN_HOME}/build"
+MVN_DOWNLOAD_DIR="${DOWNLOAD_DIR}/.mvn"
+
+# Global variable for Maven binary path
+MVN_BIN=""
+
+# Read Maven version from pom.xml if not set
+install_mvn() {
+ # Check for system Maven first
+ local SYSTEM_MVN
+ SYSTEM_MVN="$(command -v mvn)"
+
+ if [ -n "$SYSTEM_MVN" ]; then
+ local MVN_DETECTED_VERSION
+ MVN_DETECTED_VERSION="$(mvn --version 2>/dev/null | grep '^Apache
Maven' | awk '{print $3}')"
+ echo "Using system Maven: $SYSTEM_MVN (version $MVN_DETECTED_VERSION)"
>&2
+ MVN_BIN="$SYSTEM_MVN"
+ return 0
+ fi
+
+ # Get Maven version from pom.xml using grep and sed
+ local VERSION
+ VERSION=$(grep "<maven.version>" "${GLUTEN_HOME}/pom.xml" | head -n1 | \
+ awk -F '[<>]' '{print $3}')
+
+ if [ -z "$VERSION" ]; then
+ echo "ERROR: Could not read maven.version from pom.xml. Please define
<maven.version> in ${GLUTEN_HOME}/pom.xml" >&2
+ exit 1
+ fi
+
+ local MVN_LOCAL_BIN="${MVN_DOWNLOAD_DIR}/apache-maven-${VERSION}/bin/mvn"
+
+ if [ ! -f "${MVN_LOCAL_BIN}" ]; then
+ echo "Maven ${VERSION} not found locally. Downloading..." >&2
+
+ # Create download directory
+ mkdir -p "${MVN_DOWNLOAD_DIR}"
+
+ local MVN_TAR="${MVN_DOWNLOAD_DIR}/apache-maven-${VERSION}-bin.tar.gz"
+
+ if [ ! -f "${MVN_TAR}" ]; then
+ # Construct download URL using Apache closer.lua for better mirror
selection
+ local
APACHE_MIRROR="${APACHE_MIRROR:-https://www.apache.org/dyn/closer.lua}";
+ local
MVN_DOWNLOAD_PATH="maven/maven-3/${VERSION}/binaries/apache-maven-${VERSION}-bin.tar.gz"
+
+ # Try Apache closer.lua first (redirects to nearest mirror)
+ local
DOWNLOAD_URL="${APACHE_MIRROR}?action=download&filename=${MVN_DOWNLOAD_PATH}"
+
+ echo "Downloading Maven ${VERSION} from Apache mirror service..."
>&2
+ echo "URL: ${DOWNLOAD_URL}" >&2
+
+ if command -v curl > /dev/null 2>&1; then
+ # Use -L to follow redirects from closer.lua
+ if ! curl -f -L --retry 3 --retry-delay 3 \
+ --connect-timeout 30 --max-time 600 \
+ -o "${MVN_TAR}" "${DOWNLOAD_URL}"; then
+ echo "Download failed via closer.lua, trying
archive.apache.org..." >&2
+ # Fallback to archive if closer.lua fails
+ local
FALLBACK_URL="https://archive.apache.org/dist/${MVN_DOWNLOAD_PATH}";
+ curl -f -L --retry 3 --retry-delay 3 \
+ --connect-timeout 30 --max-time 600 \
+ -o "${MVN_TAR}" "${FALLBACK_URL}" || {
+ echo "ERROR: Failed to download Maven from
${FALLBACK_URL}" >&2
+ rm -f "${MVN_TAR}"
+ exit 1
+ }
+ fi
+ elif command -v wget > /dev/null 2>&1; then
+ # wget follows redirects by default
+ if ! wget --tries=3 --waitretry=3 \
+ --connect-timeout=30 --read-timeout=600 \
+ -O "${MVN_TAR}" "${DOWNLOAD_URL}"; then
+ echo "Download failed via closer.lua, trying
archive.apache.org..." >&2
+ local
FALLBACK_URL="https://archive.apache.org/dist/${MVN_DOWNLOAD_PATH}";
+ wget --tries=3 --waitretry=3 \
+ --connect-timeout=30 --read-timeout=600 \
+ -O "${MVN_TAR}" "${FALLBACK_URL}" || {
+ echo "ERROR: Failed to download Maven from
${FALLBACK_URL}" >&2
+ rm -f "${MVN_TAR}"
+ exit 1
+ }
+ fi
+ else
+ echo "ERROR: Neither curl nor wget found. Please install one
of them or install Maven manually." >&2
+ exit 1
+ fi
+
+ echo "Download completed successfully" >&2
+ fi
+
+ # Extract Maven
+ echo "Extracting Maven to ${MVN_DOWNLOAD_DIR}..." >&2
+ if ! tar -xzf "${MVN_TAR}" -C "${MVN_DOWNLOAD_DIR}"; then
+ echo "ERROR: Failed to extract Maven" >&2
+ rm -f "${MVN_TAR}"
+ exit 1
+ fi
+
+ # Clean up tar file
+ rm -f "${MVN_TAR}"
+
+ echo "Maven ${VERSION} installed successfully to
${MVN_DOWNLOAD_DIR}/apache-maven-${VERSION}" >&2
+ else
+ echo "Using downloaded Maven: ${MVN_LOCAL_BIN} (version ${VERSION})"
>&2
+ fi
+
+ # Set global variable
+ MVN_BIN="${MVN_LOCAL_BIN}"
+}
+
+# Install Maven if needed
+install_mvn
+
+# Verify Maven binary is set
+if [ -z "${MVN_BIN}" ]; then
+ echo "ERROR: Maven binary not found. Please install Maven or check your
installation." >&2
+ exit 1
+fi
+
+# Verify Maven binary exists
+if [ ! -f "${MVN_BIN}" ]; then
+ echo "ERROR: Maven binary does not exist: ${MVN_BIN}" >&2
+ exit 1
+fi
+
+_COMPILE_JVM_OPTS="-Xss128m -Xmx4g -XX:ReservedCodeCacheSize=1g"
+# Set any `mvn` options if not already present
+export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
+
+echo "MAVEN_OPTS: ${MAVEN_OPTS}" >&2
+
+"${MVN_BIN}" "$@"
+
+MVN_RETCODE=$?
+
+exit $MVN_RETCODE
diff --git a/dev/build-arrow.sh b/dev/build-arrow.sh
index 0fa2e35c20..281a2b1fc5 100755
--- a/dev/build-arrow.sh
+++ b/dev/build-arrow.sh
@@ -77,6 +77,9 @@ function build_arrow_cpp() {
function build_arrow_java() {
ARROW_INSTALL_DIR="${ARROW_PREFIX}/install"
+ # Use Gluten's Maven wrapper
+ MVN_CMD="${CURRENT_DIR}/../build/mvn"
+
# set default number of threads as cpu cores minus 2
if [[ "$(uname)" == "Darwin" ]]; then
physical_cpu_cores=$(sysctl -n hw.physicalcpu)
@@ -94,23 +97,23 @@ function build_arrow_java() {
pushd $ARROW_PREFIX/java
# Because arrow-bom module need the -DprocessAllModules
- mvn versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules
+ ${MVN_CMD} versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules
- mvn clean install -pl bom,maven/module-info-compiler-maven-plugin,vector
-am \
+ ${MVN_CMD} clean install -pl
bom,maven/module-info-compiler-maven-plugin,vector -am \
-DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip
-Dassembly.skipAssembly
# Arrow C Data Interface CPP libraries
- mvn generate-resources -P generate-libs-cdata-all-os
-Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \
+ ${MVN_CMD} generate-resources -P generate-libs-cdata-all-os
-Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \
-Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip
-N
# Arrow JNI Date Interface CPP libraries
export
PKG_CONFIG_PATH="${INSTALL_PREFIX}"/lib64/pkgconfig:"${INSTALL_PREFIX}"/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
- mvn generate-resources -Pgenerate-libs-jni-macos-linux -N
-Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \
+ ${MVN_CMD} generate-resources -Pgenerate-libs-jni-macos-linux -N
-Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \
-DARROW_GANDIVA=OFF -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF -DARROW_ORC=OFF
-DARROW_JAVA_JNI_ENABLE_ORC=OFF \
-Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip
-Dcheckstyle.skip -N
# Arrow Java libraries
- mvn install -Parrow-jni -P arrow-c-data -pl c,dataset -am \
+ ${MVN_CMD} install -Parrow-jni -P arrow-c-data -pl c,dataset -am \
-Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib
-Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib
-Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \
-Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip
-Dassembly.skipAssembly
popd
diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh
index b4b9f0ede7..be70b6f8d4 100755
--- a/dev/buildbundle-veloxbe.sh
+++ b/dev/buildbundle-veloxbe.sh
@@ -15,24 +15,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-BASEDIR=$(dirname $0)
+BASEDIR=$(dirname "$0")
source "$BASEDIR/builddeps-veloxbe.sh"
+# Use Gluten's Maven wrapper
+MVN_CMD="${BASEDIR}/../build/mvn"
+
function build_for_spark {
spark_version=$1
if [ $spark_version = "4.0" ]; then
- mvn clean install -Pbackends-velox -Pspark-$spark_version -Pjava-17
-Pscala-2.13 -DskipTests
+ ${MVN_CMD} clean install -Pbackends-velox -Pspark-$spark_version -Pjava-17
-Pscala-2.13 -DskipTests
else
- mvn clean install -Pbackends-velox -Pspark-$spark_version -DskipTests
+ ${MVN_CMD} clean install -Pbackends-velox -Pspark-$spark_version
-DskipTests
fi
}
function check_supported {
- PLATFORM=$(mvn help:evaluate -Dexpression=platform -q -DforceStdout)
- ARCH=$(mvn help:evaluate -Dexpression=arch -q -DforceStdout)
+ PLATFORM=$(${MVN_CMD} help:evaluate -Dexpression=platform -q -DforceStdout)
+ ARCH=$(${MVN_CMD} help:evaluate -Dexpression=arch -q -DforceStdout)
if [ "$PLATFORM" == "null object or invalid expression" ] || [ "$ARCH" ==
"null object or invalid expression" ]; then
- OS_NAME=$(mvn help:evaluate -Dexpression=os.name -q -DforceStdout)
- OS_ARCH=$(mvn help:evaluate -Dexpression=os.arch -q -DforceStdout)
+ OS_NAME=$(${MVN_CMD} help:evaluate -Dexpression=os.name -q -DforceStdout)
+ OS_ARCH=$(${MVN_CMD} help:evaluate -Dexpression=os.arch -q -DforceStdout)
echo "$OS_NAME-$OS_ARCH is not supported by current Gluten build."
exit 1
fi
diff --git a/dev/format-scala-code.sh b/dev/format-scala-code.sh
index 9dc7f2f220..96a782405b 100755
--- a/dev/format-scala-code.sh
+++ b/dev/format-scala-code.sh
@@ -15,6 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+BASEDIR=$(dirname "$0")
+# Use Gluten's Maven wrapper
+MVN_CMD="${BASEDIR}/../build/mvn"
+
# If a new profile is introduced for new modules, please add it here to ensure
# the new modules are covered.
PROFILES="-Pbackends-velox -Pceleborn,uniffle -Piceberg,delta,hudi,paimon \
@@ -24,10 +28,10 @@ COMMAND=$1
if [[ "$COMMAND" == "check" ]]; then
echo "Checking Scala code style.."
- mvn -q spotless:check $PROFILES
+ ${MVN_CMD} -q spotless:check $PROFILES
elif [[ "$COMMAND" == "apply" ]] || [[ "$COMMAND" == "" ]]; then
echo "Fixing Scala code style.."
- mvn -q spotless:apply $PROFILES
+ ${MVN_CMD} -q spotless:apply $PROFILES
else
echo "Unrecognized option."
exit 1
diff --git a/dev/gen-all-config-docs.sh b/dev/gen-all-config-docs.sh
index ab4e2e6ffe..2312cf6d6d 100755
--- a/dev/gen-all-config-docs.sh
+++ b/dev/gen-all-config-docs.sh
@@ -16,8 +16,12 @@
# limitations under the License.
#
+BASEDIR=$(dirname "$0")
+# Use Gluten's Maven wrapper
+MVN_CMD="${BASEDIR}/../build/mvn"
+
GLUTEN_UPDATE="${GLUTEN_UPDATE:-1}" \
-mvn clean test \
+${MVN_CMD} clean test \
-Pbackends-velox -pl backends-velox -am \
-Dtest=none \
-DfailIfNoTests=false \
diff --git a/pom.xml b/pom.xml
index f4e30677a1..d84df1236d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,8 @@
<!-- For unit tests -->
<fasterxml.version>2.15.0</fasterxml.version>
<junit.version>4.13.1</junit.version>
+ <!-- Maven version for build/mvn wrapper script -->
+ <maven.version>3.9.6</maven.version>
<junit5.version>5.11.4</junit5.version>
<flexmark.version>0.62.2</flexmark.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
