This is an automated email from the ASF dual-hosted git repository.

marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 97b12bc980 [VL] Fix and enable native UDF cluster suite (#13530)
97b12bc980 is described below

commit 97b12bc980d3e542973607462fc9b98d30a03da0
Author: kevinwilfong <[email protected]>
AuthorDate: Wed Nov 11 11:46:17 2026 -0800

    [VL] Fix and enable native UDF cluster suite (#13530)
---
 .github/workflows/velox_backend_arm.yml            | 10 ----
 .github/workflows/velox_backend_x86.yml            | 22 +++++--
 .../apache/gluten/expression/VeloxUdfSuite.scala   | 68 ++++++++++++++--------
 3 files changed, 59 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/velox_backend_arm.yml 
b/.github/workflows/velox_backend_arm.yml
index a35c3a1b29..fcdec8ca26 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -235,16 +235,6 @@ jobs:
           --conf $(realpath 
backends-velox/generated-native-benchmark/conf_12_0_*.ini) \
           --plan $(realpath 
backends-velox/generated-native-benchmark/plan_12_0_*.json) \
           --data $(realpath 
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath 
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
-      # - name: Run UDF test
-      #   run: |
-      #     # Depends on --build_example=ON.
-      #     yum update -y
-      #     yum install -y java-17-openjdk-devel
-      #     export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
-      #     export PATH=$JAVA_HOME/bin:$PATH
-      #     java -version
-      #     $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg 
-Pdelta -Ppaimon -DtagsToExclude=None \
-      #     -DtagsToInclude=org.apache.gluten.tags.UDFTest
       - name: Upload test report
         uses: actions/upload-artifact@v7
         with:
diff --git a/.github/workflows/velox_backend_x86.yml 
b/.github/workflows/velox_backend_x86.yml
index 1fda3e1528..06bfbb8b89 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -107,6 +107,7 @@ jobs:
       full_run: ${{ steps.filter.outputs.full_run }}
       cpp:      ${{ steps.filter.outputs.cpp }}
       java:     ${{ steps.filter.outputs.java }}
+      udf:      ${{ steps.filter.outputs.udf }}
       shims34:  ${{ steps.filter.outputs.shims34 }}
       shims35:  ${{ steps.filter.outputs.shims35 }}
       shims40:  ${{ steps.filter.outputs.shims40 }}
@@ -133,7 +134,7 @@ jobs:
               recent=$(git log --oneline --after="24 hours ago" HEAD | wc -l)
               if [ "$recent" -eq 0 ]; then
                 echo "No commits in the last 24 hours -- skipping nightly run."
-                for flag in cpp java shims34 shims35 shims40 shims41 shims42 
tools_it; do
+                for flag in cpp java udf shims34 shims35 shims40 shims41 
shims42 tools_it; do
                   echo "$flag=false" >> $GITHUB_OUTPUT
                 done
                 echo "full_run=false" >> $GITHUB_OUTPUT
@@ -141,7 +142,7 @@ jobs:
               fi
             fi
             # Run everything, including the extended matrices PR runs skip.
-            for flag in cpp java shims34 shims35 shims40 shims41 shims42 
tools_it; do
+            for flag in cpp java udf shims34 shims35 shims40 shims41 shims42 
tools_it; do
               echo "$flag=true" >> $GITHUB_OUTPUT
             done
             echo "full_run=true" >> $GITHUB_OUTPUT
@@ -157,6 +158,7 @@ jobs:
 
           echo "cpp=$(match      '^(cpp/|ep/build-velox/|dev/)')"              
                              >> $GITHUB_OUTPUT
           echo "java=$(match     
'^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi|paimon)/|gluten-ut/(common/|test/|src-iceberg-upstream/|pom\.xml)|package/|build/mvn)')"
 >> $GITHUB_OUTPUT
+          echo "udf=$(match      
'^(backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite\.scala|\.github/workflows/velox_backend_x86\.yml)$')"
 >> $GITHUB_OUTPUT
           echo "shims34=$(match  
'^(shims/(spark34|common)/|gluten-ut/spark34/)')"                           >> 
$GITHUB_OUTPUT
           echo "shims35=$(match  
'^(shims/(spark35|common)/|gluten-ut/spark35/)')"                           >> 
$GITHUB_OUTPUT
           echo "shims40=$(match  
'^(shims/(spark40|common)/|gluten-ut/spark40/)')"                           >> 
$GITHUB_OUTPUT
@@ -1061,7 +1063,7 @@ jobs:
 
   cpp-test-udf-test:
     needs: detect-changes
-    if: needs.detect-changes.outputs.cpp == 'true'
+    if: needs.detect-changes.outputs.cpp == 'true' || 
needs.detect-changes.outputs.udf == 'true'
     runs-on: ubuntu-22.04
     container: apache/gluten:centos-9-jdk8
     steps:
@@ -1145,14 +1147,22 @@ jobs:
           --data $(realpath 
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath 
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
       - name: Run UDF test
         run: |
+          export SPARK_SCALA_VERSION=2.12
           yum install -y java-17-openjdk-devel
           export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
           export PATH=$JAVA_HOME/bin:$PATH
           java -version
           # Depends on --build_example=ON.
-          $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg 
-Pdelta -Ppaimon -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \
-          -DtagsToInclude=org.apache.gluten.tags.UDFTest \
-          -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
+          $MVN_CMD install -Pspark-3.5 -Pbackends-velox -Pjava-17 -pl 
backends-velox -am -DskipTests
+          for bypass_registration in false true; do
+            for suite in VeloxUdfSuiteLocal VeloxUdfSuiteCluster; do
+              echo "Running $suite with bypass 
registration=$bypass_registration"
+              $MVN_CMD test -pl backends-velox -Pspark-3.5 -Pbackends-velox 
-Pjava-17 -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \
+                -DtagsToInclude=org.apache.gluten.tags.UDFTest \
+                -Dsuites=org.apache.gluten.expression.$suite \
+                -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ 
-Dspark.gluten.sql.columnar.backend.velox.nativeUDF.bypassRegistration=$bypass_registration"
+            done
+          done
       - name: Run CPP benchmark test for Spark 4.0
         run: |
           export SPARK_SCALA_VERSION=2.13
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
index aba3044e4e..19323bde75 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
@@ -20,16 +20,18 @@ import 
org.apache.gluten.backendsapi.velox.VeloxBackendSettings
 import org.apache.gluten.config.VeloxConfig
 import org.apache.gluten.execution.ProjectExecTransformer
 import org.apache.gluten.execution.WindowExecTransformer
-import org.apache.gluten.tags.{SkipTest, UDFTest}
+import org.apache.gluten.tags.UDFTest
 
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{GlutenQueryTest, Row, SparkSession}
+import org.apache.spark.sql.{AnalysisException, GlutenQueryTest, Row, 
SparkSession}
 import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.execution.window.WindowExec
 import org.apache.spark.sql.expression.UDFResolver
 
+import org.scalactic.source.Position
+
 import java.nio.file.Paths
 
 abstract class VeloxUdfSuite extends GlutenQueryTest with SQLHelper {
@@ -38,6 +40,16 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
 
   private var _spark: SparkSession = _
 
+  private lazy val bypassRegistration: Boolean =
+    new SparkConf().getBoolean(VeloxConfig.NATIVE_UDF_BYPASS_REGISTRATION.key, 
false)
+
+  private def testBypassRegistration(testName: String)(testFun: => 
Any)(implicit
+      pos: Position): Unit = {
+    if (bypassRegistration) {
+      test(testName)(testFun)
+    }
+  }
+
   // This property is used for unit tests.
   val UDFLibPathProperty: String = "velox.udf.lib.path"
 
@@ -297,7 +309,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
     }
   }
 
-  test("native udf with a plain name is callable without a hive udf class") {
+  testBypassRegistration("native udf with a plain name is callable without a 
hive udf class") {
     // No CREATE TEMPORARY FUNCTION and no Java class: the session extension 
put the name in
     // Spark's registry via SparkInjector.injectFunction, which is the only 
writer for it.
     assert(
@@ -310,7 +322,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
     checkAnswer(df, Seq(Row(2L), Row(3L), Row(4L)))
   }
 
-  test("native udf with a plain name fails at analysis when gluten is 
disabled") {
+  testBypassRegistration("native udf with a plain name fails at analysis when 
gluten is disabled") {
     withSQLConf(("spark.gluten.enabled", "false")) {
       val e = intercept[Exception] {
         spark.sql("SELECT myudf_plus_one(col1) FROM VALUES (1L) AS 
t(col1)").collect()
@@ -321,7 +333,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
     }
   }
 
-  test("native udf with a named row argument is callable") {
+  testBypassRegistration("native udf with a named row argument is callable") {
     // given
     val query = "SELECT myudf_sum_row(named_struct('a', col1, 'b', col2)) " +
       "FROM VALUES (1L, 2L), (3L, 4L) AS t(col1, col2)"
@@ -335,7 +347,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
     checkAnswer(df, expected)
   }
 
-  test("native udf returning a named row exposes its field names") {
+  testBypassRegistration("native udf returning a named row exposes its field 
names") {
     // given
     val query = "SELECT myudf_make_row(col1).b FROM VALUES (1L), (2L) AS 
t(col1)"
     val expected = Seq(Row(2L), Row(3L))
@@ -347,6 +359,28 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with 
SQLHelper {
     checkGlutenPlan[ProjectExecTransformer](df)
     checkAnswer(df, expected)
   }
+
+  test("native udf names are not registered when bypass registration is 
disabled") {
+    if (!bypassRegistration) {
+      val queries = Seq(
+        "myudf_plus_one" -> "SELECT myudf_plus_one(1L)",
+        "myudf_sum_row" -> "SELECT myudf_sum_row(named_struct('a', 1L, 'b', 
2L))",
+        "myudf_make_row" -> "SELECT myudf_make_row(1L).b"
+      )
+      queries.foreach {
+        case (name, query) =>
+          assert(
+            spark.sessionState.functionRegistry
+              .lookupFunction(FunctionIdentifier(name))
+              .isEmpty)
+          val e = intercept[AnalysisException] {
+            spark.sql(query)
+          }
+          assert(e.getMessage.contains(name))
+      }
+    }
+  }
+
 }
 
 @UDFTest
@@ -358,8 +392,6 @@ class VeloxUdfSuiteLocal extends VeloxUdfSuite {
       .set("spark.files", udfLibPath)
       .set(VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, udfLibRelativePath)
       .set("spark.shuffle.manager", 
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
-      // Off by default, so the by-name tests below have to opt in.
-      .set(VeloxConfig.NATIVE_UDF_BYPASS_REGISTRATION.key, "true")
   }
 }
 
@@ -369,35 +401,21 @@ class VeloxUdfSuiteLocal extends VeloxUdfSuite {
 //
 // VM options:
 // -Dspark.test.home=${SPARK_HOME}
-// -Dgluten.package.jar=\
-// /path/to/gluten/package/target/gluten-package-${project.version}.jar
 // -Dvelox.udf.lib.path=\
 // /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-@SkipTest
+@UDFTest
 class VeloxUdfSuiteCluster extends VeloxUdfSuite {
 
   override val master: String = "local-cluster[2,2,1024]"
 
-  val GLUTEN_JAR: String = "gluten.package.jar"
-
-  private lazy val glutenJar = sys.props.get(GLUTEN_JAR) match {
-    case Some(jar) => jar
-    case None =>
-      throw new IllegalArgumentException(
-        GLUTEN_JAR + s" cannot be null. You may set it by adding " +
-          s"-D$GLUTEN_JAR=" +
-          
"/path/to/gluten/package/target/gluten-package-${project.version}.jar")
-  }
-
   private lazy val driverUdfLibPath =
-    udfLibPath.split(",").map("file://" + _).mkString(",")
+    udfLibPath.split(",").map(p => 
Paths.get(p).toAbsolutePath.toUri.toString).mkString(",")
 
   override protected def sparkConf: SparkConf = {
     super.sparkConf
       .set("spark.files", udfLibPath)
       .set(VeloxBackendSettings.GLUTEN_VELOX_DRIVER_UDF_LIB_PATHS, 
driverUdfLibPath)
       .set(VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, udfLibRelativePath)
-      .set("spark.driver.extraClassPath", glutenJar)
-      .set("spark.executor.extraClassPath", glutenJar)
+      .set("spark.executor.extraClassPath", sys.props("java.class.path"))
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to