This is an automated email from the ASF dual-hosted git repository.

philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new ca9b6dadf4 [VL] Offload transform and filter lambdas that take an 
index argument (#13600)
ca9b6dadf4 is described below

commit ca9b6dadf47e18624000b1737ebfd79a5a5cb583
Author: Pedrum Jalali <[email protected]>
AuthorDate: Tue Nov 10 12:37:11 2026 -0800

    [VL] Offload transform and filter lambdas that take an index argument 
(#13600)
---
 .../backendsapi/velox/VeloxSparkPlanExecApi.scala  | 24 +++-------------------
 .../functions/ScalarFunctionsValidateSuite.scala   | 10 ++++++++-
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 0377d23c72..087753c21f 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -216,9 +216,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       // Transformer for array_compact.
       case LambdaFunction(_: IsNotNull, _, _) =>
         GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT, 
Seq(argument), expr)
-      case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
-        throw new GlutenNotSupportException(
-          "filter on array with lambda using index argument is not supported 
yet")
       case _ => GenericExpressionTransformer(substraitExprName, Seq(argument, 
function), expr)
     }
   }
@@ -229,12 +226,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       argument: ExpressionTransformer,
       function: ExpressionTransformer,
       expr: ArrayForAll): ExpressionTransformer = {
-    expr.function match {
-      case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
-        throw new GlutenNotSupportException(
-          "forall on array with lambda using index argument is not supported 
yet")
-      case _ => GenericExpressionTransformer(substraitExprName, Seq(argument, 
function), expr)
-    }
+    GenericExpressionTransformer(substraitExprName, Seq(argument, function), 
expr)
   }
 
   override def genArraySortTransformer(
@@ -251,12 +243,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       argument: ExpressionTransformer,
       function: ExpressionTransformer,
       expr: ArrayExists): ExpressionTransformer = {
-    expr.function match {
-      case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
-        throw new GlutenNotSupportException(
-          "exists on array with lambda using index argument is not supported 
yet")
-      case _ => GenericExpressionTransformer(substraitExprName, Seq(argument, 
function), expr)
-    }
+    GenericExpressionTransformer(substraitExprName, Seq(argument, function), 
expr)
   }
 
   /** Transform array transform to Substrait. */
@@ -265,12 +252,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       argument: ExpressionTransformer,
       function: ExpressionTransformer,
       expr: ArrayTransform): ExpressionTransformer = {
-    expr.function match {
-      case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
-        throw new GlutenNotSupportException(
-          "transform on array with lambda using index argument is not 
supported yet")
-      case _ => GenericExpressionTransformer(substraitExprName, Seq(argument, 
function), expr)
-    }
+    GenericExpressionTransformer(substraitExprName, Seq(argument, function), 
expr)
   }
 
   /** Transform posexplode to Substrait. */
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 2a2ade72f0..48bcd02e6b 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -1072,6 +1072,11 @@ class ScalarFunctionsValidateSuite extends 
FunctionsValidateSuite {
         runQueryAndCompare("select filter(value, x -> x is not null) as res 
from array_tbl;") {
           checkGlutenPlan[ProjectExecTransformer]
         }
+
+        runQueryAndCompare(
+          "select filter(value, (x, i) -> i < size(value) - 1) as res from 
array_tbl;") {
+          checkGlutenPlan[ProjectExecTransformer]
+        }
     }
   }
 
@@ -1095,10 +1100,13 @@ class ScalarFunctionsValidateSuite extends 
FunctionsValidateSuite {
   test("transform") {
     withTable("t") {
       sql("create table t (arr ARRAY<INT>) using parquet")
-      sql("insert into t values(array(1, 2, 3, null))")
+      sql("insert into t values (array(1, 2, 3, null)), (array()), (null)")
       runQueryAndCompare("select transform(arr, x -> x + 1) from t") {
         checkGlutenPlan[ProjectExecTransformer]
       }
+      runQueryAndCompare("select transform(arr, (x, i) -> x + i) from t") {
+        checkGlutenPlan[ProjectExecTransformer]
+      }
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to