This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d2e7f4b580 [VL] Refactor get_velox.sh for better readability (#10647)
d2e7f4b580 is described below
commit d2e7f4b58017f367b234666f62888def42e07fed
Author: Zouxxyy <[email protected]>
AuthorDate: Fri Oct 3 01:56:16 2025 +0800
[VL] Refactor get_velox.sh for better readability (#10647)
---
dev/builddeps-veloxbe.sh | 49 +++++++++---------
ep/build-velox/src/get_velox.sh | 107 +++++++++++++++++-----------------------
2 files changed, 71 insertions(+), 85 deletions(-)
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 2df2c94e35..8bdc72b1d3 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -18,7 +18,7 @@
####################################################################################################
# The main function of this script is to allow developers to build the
environment with one click #
# Recommended commands for first-time installation:
#
-# ./dev/buildbundle-veloxbe.sh
#
+# ./dev/buildbundle-veloxbe.sh
#
####################################################################################################
set -exu
@@ -272,29 +272,28 @@ function setup_dependencies {
source $GLUTEN_DIR/dev/build_helper_functions.sh
source ${VELOX_HOME}/scripts/setup-common.sh
- if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
- echo "Start to install dependencies"
- pushd $VELOX_HOME
- if [ $OS == 'Linux' ]; then
- setup_linux
- elif [ $OS == 'Darwin' ]; then
- setup_macos
- else
- echo "Unsupported kernel: $OS"
- exit 1
- fi
- if [ $ENABLE_S3 == "ON" ]; then
- install_aws_deps
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- install_gcs-sdk-cpp
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
- install_azure-storage-sdk-cpp
- fi
- popd
+
+ echo "Start to install dependencies"
+ pushd $VELOX_HOME
+ if [ $OS == 'Linux' ]; then
+ setup_linux
+ elif [ $OS == 'Darwin' ]; then
+ setup_macos
+ else
+ echo "Unsupported kernel: $OS"
+ exit 1
+ fi
+ if [ $ENABLE_S3 == "ON" ]; then
+ install_aws_deps
+ fi
+ if [ $ENABLE_GCS == "ON" ]; then
+ install_gcs-sdk-cpp
fi
+ if [ $ENABLE_ABFS == "ON" ]; then
+ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
+ install_azure-storage-sdk-cpp
+ fi
+ popd
}
OS=`uname -s`
@@ -303,7 +302,9 @@ commands_to_run=${OTHER_ARGUMENTS:-}
(
if [[ "x$commands_to_run" == "x" ]]; then
get_velox
- setup_dependencies
+ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ];
then
+ setup_dependencies
+ fi
build_velox_backend
else
echo "Commands to run: $commands_to_run"
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 38c62cf8ce..a9785ba842 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -16,6 +16,7 @@
set -exu
+CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
VELOX_REPO=https://github.com/oap-project/velox.git
VELOX_BRANCH=2025_10_01
VELOX_HOME=""
@@ -62,18 +63,9 @@ for arg in "$@"; do
esac
done
-function ensure_pattern_matched {
- if [ $# -ne 2 ]; then
- echo "Exactly 2 arguments are required."
- return 1
- fi
- pattern=$1
- file=$2
- matched_lines=$(grep -c "$pattern" $file)
- if [ $matched_lines -eq 0 ]; then
- return 1
- fi
-}
+if [ "$VELOX_HOME" == "" ]; then
+ VELOX_HOME="$CURRENT_DIR/../build/velox_ep"
+fi
function process_setup_ubuntu {
echo "Using setup script from Velox"
@@ -99,52 +91,43 @@ function process_setup_tencentos32 {
sed -i "/^[[:space:]]*#/!s/.*dnf config-manager --set-enabled
powertools/#&/" ${CURRENT_DIR}/setup-centos8.sh
}
-echo "Preparing Velox source code..."
-
-CURRENT_DIR=$(
- cd "$(dirname "$BASH_SOURCE")"
- pwd
-)
-
-if [ "$VELOX_HOME" == "" ]; then
- VELOX_HOME="$CURRENT_DIR/../build/velox_ep"
-fi
-VELOX_SOURCE_DIR="${VELOX_HOME}"
-
-# checkout code
-TARGET_BUILD_COMMIT="$(git ls-remote $VELOX_REPO $VELOX_BRANCH | awk '{print
$1;}' | head -n 1)"
-if [ -d $VELOX_SOURCE_DIR ]; then
- echo "Velox source folder $VELOX_SOURCE_DIR already exists..."
- cd $VELOX_SOURCE_DIR
- # if velox_branch exists, check it out,
- # otherwise assume that user prepared velox source in velox_home, skip
checkout
- if [ -n "$TARGET_BUILD_COMMIT" ]; then
- git init .
- EXISTS=$(git show-ref refs/tags/build_$TARGET_BUILD_COMMIT || true)
- if [ -z "$EXISTS" ]; then
- git fetch $VELOX_REPO
$TARGET_BUILD_COMMIT:refs/tags/build_$TARGET_BUILD_COMMIT
+function prepare_velox_source_code {
+ echo "Preparing Velox source code..."
+
+ # checkout code
+ TARGET_BUILD_COMMIT="$(git ls-remote $VELOX_REPO $VELOX_BRANCH | awk '{print
$1;}' | head -n 1)"
+ if [ -d $VELOX_HOME ]; then
+ echo "Velox source folder $VELOX_HOME already exists..."
+ cd $VELOX_HOME
+ # if velox_branch exists, check it out,
+ # otherwise assume that user prepared velox source in velox_home, skip
checkout
+ if [ -n "$TARGET_BUILD_COMMIT" ]; then
+ git init .
+ EXISTS=$(git show-ref refs/tags/build_$TARGET_BUILD_COMMIT || true)
+ if [ -z "$EXISTS" ]; then
+ git fetch $VELOX_REPO
$TARGET_BUILD_COMMIT:refs/tags/build_$TARGET_BUILD_COMMIT
+ fi
+ git reset --hard HEAD
+ git checkout refs/tags/build_$TARGET_BUILD_COMMIT
+ else
+ echo "$VELOX_BRANCH can't be found in $VELOX_REPO, skipping the
download..."
fi
- git reset --hard HEAD
- git checkout refs/tags/build_$TARGET_BUILD_COMMIT
else
- echo "$VELOX_BRANCH can't be found in $VELOX_REPO, skipping the
download..."
+ git clone $VELOX_REPO -b $VELOX_BRANCH $VELOX_HOME
+ cd $VELOX_HOME
+ git checkout $TARGET_BUILD_COMMIT
fi
-else
- git clone $VELOX_REPO -b $VELOX_BRANCH $VELOX_SOURCE_DIR
- cd $VELOX_SOURCE_DIR
- git checkout $TARGET_BUILD_COMMIT
-fi
-#sync submodules
-git submodule sync --recursive
-git submodule update --init --recursive
+ #sync submodules
+ git submodule sync --recursive
+ git submodule update --init --recursive
+}
function apply_provided_velox_patch {
if [[ -n "$UPSTREAM_VELOX_PR_ID" ]]; then
echo "Applying patch for PR #$UPSTREAM_VELOX_PR_ID ..."
- local velox_home=$1
local patch_name="$UPSTREAM_VELOX_PR_ID.patch"
- pushd $velox_home
+ pushd $VELOX_HOME
rm -f $patch_name
wget -nv
"https://patch-diff.githubusercontent.com/raw/facebookincubator/velox/pull/$UPSTREAM_VELOX_PR_ID.patch";
\
-O "$patch_name" || {
@@ -160,15 +143,11 @@ function apply_provided_velox_patch {
}
function apply_compilation_fixes {
- current_dir=$1
- velox_home=$2
-
- sudo cp ${current_dir}/modify_arrow.patch
${velox_home}/CMake/resolve_dependency_modules/arrow/
- sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch
${velox_home}/CMake/resolve_dependency_modules/arrow/
-
- git add
${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to
avoid the file from being deleted by git clean -dffx :/
- git add
${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow_dataset_scan_option.patch
# to avoid the file from being deleted by git clean -dffx :/
+ sudo cp ${CURRENT_DIR}/modify_arrow.patch
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/
+ sudo cp ${CURRENT_DIR}/modify_arrow_dataset_scan_option.patch
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/
+ git add
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to
avoid the file from being deleted by git clean -dffx :/
+ git add
${VELOX_HOME}/CMake/resolve_dependency_modules/arrow/modify_arrow_dataset_scan_option.patch
# to avoid the file from being deleted by git clean -dffx :/
}
function setup_linux {
@@ -231,9 +210,7 @@ function setup_linux {
fi
}
-apply_provided_velox_patch $VELOX_SOURCE_DIR
-
-if [[ "$RUN_SETUP_SCRIPT" == "ON" ]]; then
+function apply_setup_fixes() {
if [ $OS == 'Linux' ]; then
setup_linux
elif [ $OS == 'Darwin' ]; then
@@ -242,8 +219,16 @@ if [[ "$RUN_SETUP_SCRIPT" == "ON" ]]; then
echo "Unsupported kernel: $OS"
exit 1
fi
+}
+
+prepare_velox_source_code
+
+if [[ "$RUN_SETUP_SCRIPT" == "ON" ]]; then
+ apply_setup_fixes
fi
-apply_compilation_fixes $CURRENT_DIR $VELOX_SOURCE_DIR
+apply_provided_velox_patch
+
+apply_compilation_fixes
echo "Finished getting Velox code"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
