This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 97b12bc980 [VL] Fix and enable native UDF cluster suite (#13530)
97b12bc980 is described below
commit 97b12bc980d3e542973607462fc9b98d30a03da0
Author: kevinwilfong <[email protected]>
AuthorDate: Wed Nov 11 11:46:17 2026 -0800
[VL] Fix and enable native UDF cluster suite (#13530)
---
.github/workflows/velox_backend_arm.yml | 10 ----
.github/workflows/velox_backend_x86.yml | 22 +++++--
.../apache/gluten/expression/VeloxUdfSuite.scala | 68 ++++++++++++++--------
3 files changed, 59 insertions(+), 41 deletions(-)
diff --git a/.github/workflows/velox_backend_arm.yml
b/.github/workflows/velox_backend_arm.yml
index a35c3a1b29..fcdec8ca26 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -235,16 +235,6 @@ jobs:
--conf $(realpath
backends-velox/generated-native-benchmark/conf_12_0_*.ini) \
--plan $(realpath
backends-velox/generated-native-benchmark/plan_12_0_*.json) \
--data $(realpath
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
- # - name: Run UDF test
- # run: |
- # # Depends on --build_example=ON.
- # yum update -y
- # yum install -y java-17-openjdk-devel
- # export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
- # export PATH=$JAVA_HOME/bin:$PATH
- # java -version
- # $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg
-Pdelta -Ppaimon -DtagsToExclude=None \
- # -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload test report
uses: actions/upload-artifact@v7
with:
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 1fda3e1528..06bfbb8b89 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -107,6 +107,7 @@ jobs:
full_run: ${{ steps.filter.outputs.full_run }}
cpp: ${{ steps.filter.outputs.cpp }}
java: ${{ steps.filter.outputs.java }}
+ udf: ${{ steps.filter.outputs.udf }}
shims34: ${{ steps.filter.outputs.shims34 }}
shims35: ${{ steps.filter.outputs.shims35 }}
shims40: ${{ steps.filter.outputs.shims40 }}
@@ -133,7 +134,7 @@ jobs:
recent=$(git log --oneline --after="24 hours ago" HEAD | wc -l)
if [ "$recent" -eq 0 ]; then
echo "No commits in the last 24 hours -- skipping nightly run."
- for flag in cpp java shims34 shims35 shims40 shims41 shims42
tools_it; do
+ for flag in cpp java udf shims34 shims35 shims40 shims41
shims42 tools_it; do
echo "$flag=false" >> $GITHUB_OUTPUT
done
echo "full_run=false" >> $GITHUB_OUTPUT
@@ -141,7 +142,7 @@ jobs:
fi
fi
# Run everything, including the extended matrices PR runs skip.
- for flag in cpp java shims34 shims35 shims40 shims41 shims42
tools_it; do
+ for flag in cpp java udf shims34 shims35 shims40 shims41 shims42
tools_it; do
echo "$flag=true" >> $GITHUB_OUTPUT
done
echo "full_run=true" >> $GITHUB_OUTPUT
@@ -157,6 +158,7 @@ jobs:
echo "cpp=$(match '^(cpp/|ep/build-velox/|dev/)')"
>> $GITHUB_OUTPUT
echo "java=$(match
'^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi|paimon)/|gluten-ut/(common/|test/|src-iceberg-upstream/|pom\.xml)|package/|build/mvn)')"
>> $GITHUB_OUTPUT
+ echo "udf=$(match
'^(backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite\.scala|\.github/workflows/velox_backend_x86\.yml)$')"
>> $GITHUB_OUTPUT
echo "shims34=$(match
'^(shims/(spark34|common)/|gluten-ut/spark34/)')" >>
$GITHUB_OUTPUT
echo "shims35=$(match
'^(shims/(spark35|common)/|gluten-ut/spark35/)')" >>
$GITHUB_OUTPUT
echo "shims40=$(match
'^(shims/(spark40|common)/|gluten-ut/spark40/)')" >>
$GITHUB_OUTPUT
@@ -1061,7 +1063,7 @@ jobs:
cpp-test-udf-test:
needs: detect-changes
- if: needs.detect-changes.outputs.cpp == 'true'
+ if: needs.detect-changes.outputs.cpp == 'true' ||
needs.detect-changes.outputs.udf == 'true'
runs-on: ubuntu-22.04
container: apache/gluten:centos-9-jdk8
steps:
@@ -1145,14 +1147,22 @@ jobs:
--data $(realpath
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
- name: Run UDF test
run: |
+ export SPARK_SCALA_VERSION=2.12
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
# Depends on --build_example=ON.
- $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg
-Pdelta -Ppaimon -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \
- -DtagsToInclude=org.apache.gluten.tags.UDFTest \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
+ $MVN_CMD install -Pspark-3.5 -Pbackends-velox -Pjava-17 -pl
backends-velox -am -DskipTests
+ for bypass_registration in false true; do
+ for suite in VeloxUdfSuiteLocal VeloxUdfSuiteCluster; do
+ echo "Running $suite with bypass
registration=$bypass_registration"
+ $MVN_CMD test -pl backends-velox -Pspark-3.5 -Pbackends-velox
-Pjava-17 -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \
+ -DtagsToInclude=org.apache.gluten.tags.UDFTest \
+ -Dsuites=org.apache.gluten.expression.$suite \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.backend.velox.nativeUDF.bypassRegistration=$bypass_registration"
+ done
+ done
- name: Run CPP benchmark test for Spark 4.0
run: |
export SPARK_SCALA_VERSION=2.13
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
index aba3044e4e..19323bde75 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
@@ -20,16 +20,18 @@ import
org.apache.gluten.backendsapi.velox.VeloxBackendSettings
import org.apache.gluten.config.VeloxConfig
import org.apache.gluten.execution.ProjectExecTransformer
import org.apache.gluten.execution.WindowExecTransformer
-import org.apache.gluten.tags.{SkipTest, UDFTest}
+import org.apache.gluten.tags.UDFTest
import org.apache.spark.SparkConf
-import org.apache.spark.sql.{GlutenQueryTest, Row, SparkSession}
+import org.apache.spark.sql.{AnalysisException, GlutenQueryTest, Row,
SparkSession}
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.execution.ProjectExec
import org.apache.spark.sql.execution.window.WindowExec
import org.apache.spark.sql.expression.UDFResolver
+import org.scalactic.source.Position
+
import java.nio.file.Paths
abstract class VeloxUdfSuite extends GlutenQueryTest with SQLHelper {
@@ -38,6 +40,16 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
private var _spark: SparkSession = _
+ private lazy val bypassRegistration: Boolean =
+ new SparkConf().getBoolean(VeloxConfig.NATIVE_UDF_BYPASS_REGISTRATION.key,
false)
+
+ private def testBypassRegistration(testName: String)(testFun: =>
Any)(implicit
+ pos: Position): Unit = {
+ if (bypassRegistration) {
+ test(testName)(testFun)
+ }
+ }
+
// This property is used for unit tests.
val UDFLibPathProperty: String = "velox.udf.lib.path"
@@ -297,7 +309,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
}
}
- test("native udf with a plain name is callable without a hive udf class") {
+ testBypassRegistration("native udf with a plain name is callable without a
hive udf class") {
// No CREATE TEMPORARY FUNCTION and no Java class: the session extension
put the name in
// Spark's registry via SparkInjector.injectFunction, which is the only
writer for it.
assert(
@@ -310,7 +322,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
checkAnswer(df, Seq(Row(2L), Row(3L), Row(4L)))
}
- test("native udf with a plain name fails at analysis when gluten is
disabled") {
+ testBypassRegistration("native udf with a plain name fails at analysis when
gluten is disabled") {
withSQLConf(("spark.gluten.enabled", "false")) {
val e = intercept[Exception] {
spark.sql("SELECT myudf_plus_one(col1) FROM VALUES (1L) AS
t(col1)").collect()
@@ -321,7 +333,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
}
}
- test("native udf with a named row argument is callable") {
+ testBypassRegistration("native udf with a named row argument is callable") {
// given
val query = "SELECT myudf_sum_row(named_struct('a', col1, 'b', col2)) " +
"FROM VALUES (1L, 2L), (3L, 4L) AS t(col1, col2)"
@@ -335,7 +347,7 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
checkAnswer(df, expected)
}
- test("native udf returning a named row exposes its field names") {
+ testBypassRegistration("native udf returning a named row exposes its field
names") {
// given
val query = "SELECT myudf_make_row(col1).b FROM VALUES (1L), (2L) AS
t(col1)"
val expected = Seq(Row(2L), Row(3L))
@@ -347,6 +359,28 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with
SQLHelper {
checkGlutenPlan[ProjectExecTransformer](df)
checkAnswer(df, expected)
}
+
+ test("native udf names are not registered when bypass registration is
disabled") {
+ if (!bypassRegistration) {
+ val queries = Seq(
+ "myudf_plus_one" -> "SELECT myudf_plus_one(1L)",
+ "myudf_sum_row" -> "SELECT myudf_sum_row(named_struct('a', 1L, 'b',
2L))",
+ "myudf_make_row" -> "SELECT myudf_make_row(1L).b"
+ )
+ queries.foreach {
+ case (name, query) =>
+ assert(
+ spark.sessionState.functionRegistry
+ .lookupFunction(FunctionIdentifier(name))
+ .isEmpty)
+ val e = intercept[AnalysisException] {
+ spark.sql(query)
+ }
+ assert(e.getMessage.contains(name))
+ }
+ }
+ }
+
}
@UDFTest
@@ -358,8 +392,6 @@ class VeloxUdfSuiteLocal extends VeloxUdfSuite {
.set("spark.files", udfLibPath)
.set(VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, udfLibRelativePath)
.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
- // Off by default, so the by-name tests below have to opt in.
- .set(VeloxConfig.NATIVE_UDF_BYPASS_REGISTRATION.key, "true")
}
}
@@ -369,35 +401,21 @@ class VeloxUdfSuiteLocal extends VeloxUdfSuite {
//
// VM options:
// -Dspark.test.home=${SPARK_HOME}
-// -Dgluten.package.jar=\
-// /path/to/gluten/package/target/gluten-package-${project.version}.jar
// -Dvelox.udf.lib.path=\
// /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-@SkipTest
+@UDFTest
class VeloxUdfSuiteCluster extends VeloxUdfSuite {
override val master: String = "local-cluster[2,2,1024]"
- val GLUTEN_JAR: String = "gluten.package.jar"
-
- private lazy val glutenJar = sys.props.get(GLUTEN_JAR) match {
- case Some(jar) => jar
- case None =>
- throw new IllegalArgumentException(
- GLUTEN_JAR + s" cannot be null. You may set it by adding " +
- s"-D$GLUTEN_JAR=" +
-
"/path/to/gluten/package/target/gluten-package-${project.version}.jar")
- }
-
private lazy val driverUdfLibPath =
- udfLibPath.split(",").map("file://" + _).mkString(",")
+ udfLibPath.split(",").map(p =>
Paths.get(p).toAbsolutePath.toUri.toString).mkString(",")
override protected def sparkConf: SparkConf = {
super.sparkConf
.set("spark.files", udfLibPath)
.set(VeloxBackendSettings.GLUTEN_VELOX_DRIVER_UDF_LIB_PATHS,
driverUdfLibPath)
.set(VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, udfLibRelativePath)
- .set("spark.driver.extraClassPath", glutenJar)
- .set("spark.executor.extraClassPath", glutenJar)
+ .set("spark.executor.extraClassPath", sys.props("java.class.path"))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]