This is an automated email from the ASF dual-hosted git repository.
richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new e21bb28d [AURON-1212] Consolidate build entry: integrate
release-docker.sh into auron-build.sh (with Docker support) (#1213)
e21bb28d is described below
commit e21bb28d30fa305ded09b4f4e3874f187093849e
Author: Ruilei Ma <[email protected]>
AuthorDate: Wed Sep 3 10:56:46 2025 +0800
[AURON-1212] Consolidate build entry: integrate release-docker.sh into
auron-build.sh (with Docker support) (#1213)
* [AURON-1212] Consolidate build entry: integrate release-docker.sh into
auron-build.sh (with Docker support)
* Test fot GA
* Revert "Test fot GA"
---
.github/workflows/build-ce7-releases.yml | 14 ++++++++++++--
README.md | 26 ++++++++++++-------------
auron-build.sh | 33 ++++++++++++++++++++++++++++----
dev/docker-build/centos7/Dockerfile | 11 +----------
dev/docker-build/docker-compose.yml | 13 ++++++++-----
release-docker.sh | 24 -----------------------
6 files changed, 62 insertions(+), 59 deletions(-)
diff --git a/.github/workflows/build-ce7-releases.yml
b/.github/workflows/build-ce7-releases.yml
index 1cf9c24f..fe27aac4 100644
--- a/.github/workflows/build-ce7-releases.yml
+++ b/.github/workflows/build-ce7-releases.yml
@@ -61,6 +61,12 @@ jobs:
submodules: recursive
- uses: ./.github/actions/setup-rust-toolchain
+ - name: Extract spark version
+ run: |
+ raw="${{ matrix.sparkver }}"
+ ver="${raw#spark-}"
+ echo "sparkver_short=$ver" >> $GITHUB_ENV
+
- name: Detect osclassfier
id: detect-osclassfier
run: |
@@ -80,8 +86,12 @@ jobs:
- name: Build auron-${{ matrix.sparkver }}_${{ matrix.scalaver }}
run: |
- sed -i 's/docker-compose -f/docker compose -f/g' ./release-docker.sh
- SHIM=${{ matrix.sparkver }} MODE=release SCALA_VERSION=${{
matrix.scalaver }} ./release-docker.sh
+ sed -i 's/docker-compose -f/docker compose -f/g' ./auron-build.sh
+ ./auron-build.sh \
+ --docker true \
+ --release \
+ --sparkver ${{ env.sparkver_short }} \
+ --scalaver ${{ matrix.scalaver }}
- name: Upload auron-${{ matrix.sparkver }}_${{ matrix.scalaver }}
uses: actions/upload-artifact@v4
diff --git a/README.md b/README.md
index f09cf437..140c6f7c 100644
--- a/README.md
+++ b/README.md
@@ -43,37 +43,35 @@ Based on the inherent well-defined extensibility of
DataFusion, Auron can be eas
- Simple and Aggregate functions.
- File formats.
-We encourage you to [extend
DataFusion](https://github.com/apache/arrow-datafusion) capability directly and
add the
+We encourage you to extend
[DataFusion](https://github.com/apache/arrow-datafusion) capability directly
and add the
supports in Auron with simple modifications in plan-serde and extension
translation.
## Build from source
-To build Auron, please follow the steps below:
+To build Auron from source, follow the steps below:
1. Install Rust
-The native execution lib is written in Rust. So you're required to install
Rust (nightly) first for
-compilation. We recommend you to use [rustup](https://rustup.rs/).
+Auron's native execution lib is written in Rust. You need to install Rust
(nightly) before compiling.
+
+We recommend using [rustup](https://rustup.rs/) for installation.
2. Install JDK
-Auron has been well tested on jdk8/11/17.
+Auron has been well tested with JDK 8, 11, and 17.
+
+Make sure `JAVA_HOME` is properly set and points to your desired version.
3. Check out the source code.
4. Build the project.
-use `./auron-build.sh` for building the project. execute `./auron-build.sh
--help` for help.
-
-After the build is finished, a fat Jar package that contains all the
dependencies will be generated in the `target`
-directory.
+You can build Auron either *locally* or *inside Docker with CentOS7* using a
unified script: `auron-build.sh`.
-## Build with docker
+Run `./auron-build.sh --help` to see all available options.
-You can use the following command to build a centos-7 compatible release:
-```shell
-SHIM=spark-3.3 MODE=release JAVA_VERSION=8 SCALA_VERSION=2.12
./release-docker.sh
-```
+After the build completes, a fat JAR with all dependencies will be generated
in either the `target/` directory (for local builds)
+or `target-docker/` directory (for Docker builds), depending on the selected
build mode.
## Run Spark Job with Auron Accelerator
diff --git a/auron-build.sh b/auron-build.sh
index df4bc685..17ec9990 100755
--- a/auron-build.sh
+++ b/auron-build.sh
@@ -24,6 +24,7 @@ print_help() {
echo "Build Auron project with specified Maven profiles"
echo
echo "Options:"
+ echo " --docker <true|false> Build in Docker with CentOS7 (default:
false)"
echo " --pre Activate pre-release profile"
echo " --release Activate release profile"
echo " --sparkver <VERSION> Specify Spark version (e.g.
3.0/3.1/3.2/3.3/3.4/3.5)"
@@ -36,15 +37,15 @@ print_help() {
echo " -h, --help Show this help message"
echo
echo "Examples:"
- echo " $0 --pre --sparkver 3.5 --scalaver 2.12"
echo " $0 --pre --sparkver 3.5 --scalaver 2.12 -DskipBuildNative"
- echo " $0 --clean true --skiptests true --release --sparkver 3.5
--scalaver 2.12 --celeborn 0.5 --uniffle 0.9 --paimon 1.1"
+ echo " $0 --docker true --clean true --skiptests true --release
--sparkver 3.5 --scalaver 2.12 --celeborn 0.5 --uniffle 0.9 --paimon 1.1"
exit 0
}
MVN_CMD="$(dirname "$0")/build/mvn"
# Initialize variables
+USE_DOCKER=false
PRE_PROFILE=false
RELEASE_PROFILE=false
CLEAN=true
@@ -66,6 +67,15 @@ while [[ $# -gt 0 ]]; do
RELEASE_PROFILE=true
shift
;;
+ --docker)
+ if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then
+ USE_DOCKER="$2"
+ shift 2
+ else
+ echo "ERROR: --docker requires true/false" >&2
+ exit 1
+ fi
+ ;;
--clean)
if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then
CLEAN="$2"
@@ -213,5 +223,20 @@ fi
MVN_ARGS=("${CLEAN_ARGS[@]}" "${BUILD_ARGS[@]}")
# Execute Maven command
-echo "Compiling with maven args: $MVN_CMD ${MVN_ARGS[@]} $@"
-"$MVN_CMD" "${MVN_ARGS[@]}" "$@"
+if [[ "$USE_DOCKER" == true ]]; then
+ # In Docker mode, use multi-threaded Maven build with -T8 for faster
compilation
+ BUILD_ARGS+=("-T8")
+ if [[ "$CLEAN" == true ]]; then
+ # Clean the host-side directory that is mounted into the Docker
container.
+ # This avoids "device or resource busy" errors when running `mvn
clean` inside the container.
+ echo "[INFO] Docker mode: manually cleaning target-docker contents..."
+ rm -rf ./target-docker/* || echo "[WARN] Failed to clean
target-docker/*"
+ fi
+
+ echo "[INFO] Compiling inside Docker container..."
+ export AURON_BUILD_ARGS="${BUILD_ARGS[*]}"
+ exec docker-compose -f dev/docker-build/docker-compose.yml up
--abort-on-container-exit
+else
+ echo "[INFO] Compiling locally with maven args: $MVN_CMD ${MVN_ARGS[@]} $@"
+ "$MVN_CMD" "${MVN_ARGS[@]}" "$@"
+fi
diff --git a/dev/docker-build/centos7/Dockerfile
b/dev/docker-build/centos7/Dockerfile
index 2a848a46..9d813842 100644
--- a/dev/docker-build/centos7/Dockerfile
+++ b/dev/docker-build/centos7/Dockerfile
@@ -27,7 +27,7 @@ RUN yum install -y centos-release-scl epel-release
RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo
RUN sed -i "s/^#.*baseurl=http/baseurl=https/g" /etc/yum.repos.d/*.repo
RUN sed -i "s/^mirrorlist/#mirrorlist/g" /etc/yum.repos.d/*.repo
-RUN yum install -y libzip unzip wget cmake3 openssl-devel
+RUN yum install -y libzip unzip openssl-devel
# install gcc-11
RUN yum install -y devtoolset-11-gcc devtoolset-11-gcc-c++
@@ -41,12 +41,3 @@ RUN /rustup-init -y --default-toolchain
nightly-2023-08-01-x86_64-unknown-linux-
# install java
RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel
RUN echo 'export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"' >> ~/.bashrc
-
-# install maven
-RUN yum install -y rh-maven35
-RUN echo 'source /opt/rh/rh-maven35/enable' >> ~/.bashrc
-
-# install protoc
-RUN wget -O /protobuf-21.7-linux-x86_64.zip
https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protoc-21.7-linux-x86_64.zip
-RUN mkdir /protobuf-bin && (cd /protobuf-bin && unzip
/protobuf-21.7-linux-x86_64.zip)
-RUN echo 'export PATH="$PATH:/protobuf-bin/bin"' >> ~/.bashrc
diff --git a/dev/docker-build/docker-compose.yml
b/dev/docker-build/docker-compose.yml
index 064ba19a..20c948fd 100644
--- a/dev/docker-build/docker-compose.yml
+++ b/dev/docker-build/docker-compose.yml
@@ -33,8 +33,11 @@ services:
-
./../../target-docker/build-helper-assembly-target:/auron/dev/mvn-build-helper/assembly/target:rw
environment:
RUSTFLAGS: "-C target-cpu=skylake"
- SHIM: "${SHIM}"
- MODE: "${MODE}"
- JAVA_VERSION: "${JAVA_VERSION}"
- SCALA_VERSION: "${SCALA_VERSION}"
- command: "bash -c 'source ~/.bashrc && cd /auron && ./build/mvn -T8
package -P${SHIM} -P${MODE} -Pjdk-${JAVA_VERSION} -Pscala-${SCALA_VERSION}'"
+ AURON_BUILD_ARGS: "${AURON_BUILD_ARGS}"
+ command: >
+ bash -c '
+ source ~/.bashrc &&
+ cd /auron &&
+ echo "[DOCKER] Running: ./build/mvn $AURON_BUILD_ARGS" &&
+ ./build/mvn $AURON_BUILD_ARGS
+ '
diff --git a/release-docker.sh b/release-docker.sh
deleted file mode 100755
index bdb96557..00000000
--- a/release-docker.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SHIM="${SHIM:-spark-3.0}"
-export MODE="${MODE:-release}"
-export JAVA_VERSION="${JAVA_VERSION:-8}"
-export SCALA_VERSION="${SCALA_VERSION:-2.12}"
-
-docker-compose -f dev/docker-build/docker-compose.yml up
