This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2528d9cd5a [GLUTEN-9949][VL] Enable enhanced features compile (#9957)
2528d9cd5a is described below
commit 2528d9cd5a0f03728daaf9f123c35b99d57964ad
Author: Jin Chengcheng <[email protected]>
AuthorDate: Thu Jun 19 06:12:50 2025 +0100
[GLUTEN-9949][VL] Enable enhanced features compile (#9957)
---
.github/workflows/velox_backend_cache.yml | 28 ++++++++++++++++++++++
.github/workflows/velox_backend_x86.yml | 19 +++++++++++++++
.../org/apache/gluten/config/ConfigJniWrapper.java | 22 +++++++++++++++++
.../gluten/backendsapi/velox/VeloxBackend.scala | 2 ++
.../org/apache/gluten/config/VeloxConfig.scala | 2 ++
cpp/CMakeLists.txt | 5 ++++
cpp/velox/compute/VeloxBackend.cc | 1 -
cpp/velox/jni/VeloxJniWrapper.cc | 10 ++++++++
dev/builddeps-veloxbe.sh | 13 +++++++++-
ep/build-velox/src/get_velox.sh | 11 +++++++++
.../gluten/backendsapi/BackendSettingsApi.scala | 2 ++
11 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index 37050f9cbb..47beb8097c 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -106,6 +106,34 @@ jobs:
path: '${{ env.CCACHE_DIR }}'
key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}}
+ cache-enhanced-shared-lib-centos-8:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ ubuntu-22.04 ]
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v2
+ - name: Get Ccache
+ uses: actions/cache/restore@v3
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key:
ccache-enhanced-centos8-release-shared-${{runner.arch}}-${{github.sha}}
+ restore-keys: |
+ ccache-enhanced-centos8-release-shared-${{runner.arch}}
+ - name: Build Gluten
+ run: |
+ df -a
+ source /opt/rh/gcc-toolset-11/enable
+ bash dev/buildbundle-veloxbe.sh --run_setup_script=OFF
--build_arrow=OFF --spark_version=3.4 --enable_enhanced_features=ON
+ ccache -s
+ - name: Save Ccache
+ uses: actions/cache/save@v3
+ id: ccache
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key:
ccache-enhanced-centos8-release-shared-${{runner.arch}}-${{github.sha}}
+
cache-shared-lib-centos-9:
runs-on: ${{ matrix.os }}
strategy:
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index cfd8607ce6..9af5c1b2a6 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -1274,6 +1274,25 @@ jobs:
name: ${{ github.job }}-test-log
path: "**/target/*.log"
+ build-enhanced-feature-centos-8:
+ runs-on: ubuntu-22.04
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v2
+ - name: Get Ccache
+ uses: actions/cache/restore@v4
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key:
ccache-enhanced-centos8-release-shared-${{runner.arch}}-${{github.sha}}
+ restore-keys: |
+ ccache-enhanced-centos8-release-shared-${{runner.arch}}
+ - name: Build Gluten
+ run: |
+ df -a
+ source /opt/rh/gcc-toolset-11/enable
+ bash dev/buildbundle-veloxbe.sh --run_setup_script=OFF
--build_arrow=OFF --spark_version=3.4 --enable_enhanced_features=ON
+ ccache -s
+
build-cudf-centos-9:
runs-on: ubuntu-22.04
container: apache/gluten:centos-9-jdk8-cudf
diff --git
a/backends-velox/src/main/java/org/apache/gluten/config/ConfigJniWrapper.java
b/backends-velox/src/main/java/org/apache/gluten/config/ConfigJniWrapper.java
new file mode 100644
index 0000000000..5bc393499f
--- /dev/null
+++
b/backends-velox/src/main/java/org/apache/gluten/config/ConfigJniWrapper.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.config;
+
+public class ConfigJniWrapper {
+
+ public static native boolean isEnhancedFeaturesEnabled();
+}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index 08a7f2db85..cd5d80d3f5 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -560,4 +560,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
override def supportIcebergEqualityDeleteRead(): Boolean = false
override def reorderColumnsForPartitionWrite(): Boolean = true
+
+ override def enableEnhancedFeatures(): Boolean =
VeloxConfig.get.enableEnhancedFeatures()
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index de8eeb2f62..55c38f3041 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -68,6 +68,8 @@ class VeloxConfig(conf: SQLConf) extends GlutenConfig(conf) {
getConf(VELOX_PROPAGATE_IGNORE_NULL_KEYS_ENABLED)
def floatingPointMode: String = getConf(FLOATING_POINT_MODE)
+
+ def enableEnhancedFeatures(): Boolean =
ConfigJniWrapper.isEnhancedFeaturesEnabled
}
object VeloxConfig {
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 1eaa578d7c..87e0e42b62 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -57,6 +57,7 @@ option(ENABLE_HDFS "Enable HDFS" OFF)
option(ENABLE_ORC "Enable ORC" OFF)
option(ENABLE_ABFS "Enable ABFS" OFF)
option(ENABLE_GPU "Enable GPU" OFF)
+option(ENABLE_ENHANCED_FEATURES "Enable enhanced features" OFF)
set(root_directory ${PROJECT_BINARY_DIR})
get_filename_component(GLUTEN_HOME ${CMAKE_SOURCE_DIR} DIRECTORY)
@@ -241,6 +242,10 @@ if(ENABLE_GPU)
add_definitions(-DGLUTEN_ENABLE_GPU)
endif()
+if(ENABLE_ENHANCED_FEATURES)
+ add_definitions(-DGLUTEN_ENABLE_ENHANCED_FEATURES)
+endif()
+
# Subdirectories
add_subdirectory(core)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 10f1084912..ada113f946 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -165,7 +165,6 @@ void VeloxBackend::init(
if (backendConf_->get<bool>(kCudfEnabled, kCudfEnabledDefault)) {
velox::cudf_velox::registerCudf();
}
-
#endif
initJolFilesystem();
diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
index ca1b4b0c1f..fdadf13ba8 100644
--- a/cpp/velox/jni/VeloxJniWrapper.cc
+++ b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -580,6 +580,16 @@
Java_org_apache_gluten_vectorized_UnifflePartitionWriterJniWrapper_createPartiti
JNI_METHOD_END(kInvalidObjectHandle)
}
+JNIEXPORT jboolean JNICALL
Java_org_apache_gluten_config_ConfigJniWrapper_isEnhancedFeaturesEnabled( //
NOLINT
+ JNIEnv* env,
+ jclass) {
+#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES
+ return true;
+#else
+ return false;
+#endif
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index ef435463e8..d7548cd8fa 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -40,6 +40,7 @@ ENABLE_HDFS=OFF
ENABLE_ABFS=OFF
ENABLE_VCPKG=OFF
ENABLE_GPU=OFF
+ENABLE_ENHANCED_FEATURES=OFF
RUN_SETUP_SCRIPT=ON
VELOX_REPO=""
VELOX_BRANCH=""
@@ -121,6 +122,10 @@ do
ENABLE_GPU=("${arg#*=}")
shift # Remove argument name from processing
;;
+ --enable_enhanced_features=*)
+ ENABLE_ENHANCED_FEATURES=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
--run_setup_script=*)
RUN_SETUP_SCRIPT=("${arg#*=}")
shift # Remove argument name from processing
@@ -180,9 +185,14 @@ function concat_velox_param {
VELOX_PARAMETER+="--velox_home=$VELOX_HOME "
fi
+ if [ "$ENABLE_ENHANCED_FEATURES" = "ON" ]; then
+ VELOX_PARAMETER+="--enable_enhanced_features=$ENABLE_ENHANCED_FEATURES
"
+ fi
+
VELOX_PARAMETER+="--run_setup_script=$RUN_SETUP_SCRIPT "
}
+
if [ "$ENABLE_VCPKG" = "ON" ]; then
# vcpkg will install static depends and init build environment
BUILD_OPTIONS="--build_tests=$BUILD_TESTS --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS \
@@ -237,7 +247,8 @@ function build_gluten_cpp {
-DENABLE_S3=$ENABLE_S3 \
-DENABLE_HDFS=$ENABLE_HDFS \
-DENABLE_ABFS=$ENABLE_ABFS \
- -DENABLE_GPU=$ENABLE_GPU"
+ -DENABLE_GPU=$ENABLE_GPU \
+ -DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
if [ $OS == 'Darwin' ]; then
if [ -n "$INSTALL_PREFIX" ]; then
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 7c11712ef4..252454b1cf 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -20,6 +20,9 @@ VELOX_REPO=https://github.com/oap-project/velox.git
VELOX_BRANCH=2025_06_18
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
+VELOX_ENHANCED_REPO=https://github.com/oap-project/velox.git
+VELOX_ENHANCED_BRANCH=2025_06_10
+ENABLE_ENHANCED_FEATURES=OFF
OS=`uname -s`
@@ -27,10 +30,12 @@ for arg in "$@"; do
case $arg in
--velox_repo=*)
VELOX_REPO=("${arg#*=}")
+ VELOX_ENHANCED_REPO=("${arg#*=}")
shift # Remove argument name from processing
;;
--velox_branch=*)
VELOX_BRANCH=("${arg#*=}")
+ VELOX_ENHANCED_BRANCH=("${arg#*=}")
shift # Remove argument name from processing
;;
--velox_home=*)
@@ -41,6 +46,12 @@ for arg in "$@"; do
RUN_SETUP_SCRIPT=("${arg#*=}")
shift # Remove argument name from processing
;;
+ --enable_enhanced_features=*)
+ ENABLE_ENHANCED_FEATURES=("${arg#*=}")
+ VELOX_REPO=$VELOX_ENHANCED_REPO
+ VELOX_BRANCH=$VELOX_ENHANCED_BRANCH
+ shift # Remove argument name from processing
+ ;;
*)
OTHER_ARGUMENTS+=("$1")
shift # Remove generic argument from processing
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index c031fb3aa1..3f1075d0ac 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -157,4 +157,6 @@ trait BackendSettingsApi {
def supportIcebergEqualityDeleteRead(): Boolean = true
def reorderColumnsForPartitionWrite(): Boolean = false
+
+ def enableEnhancedFeatures(): Boolean = false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
