(incubator-gluten) branch main updated: [VL] Enable spark function map_filter (#8883)

[marong]

This is an automated email from the ASF dual-hosted git repository.

marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 81fbc0e427 [VL] Enable spark function map_filter (#8883)
81fbc0e427 is described below

commit 81fbc0e427be912cd92f7f6a361f040b93b2351f
Author: Jinhai （陈锦海） <146867566+j7n...@users.noreply.github.com>
AuthorDate: Tue Mar 4 21:21:03 2025 +0800

    [VL] Enable spark function map_filter (#8883)
---
 .../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala |  1 +
 .../gluten/execution/ScalarFunctionsValidateSuite.scala  | 16 ++++++++++++++++
 .../org/apache/gluten/expression/ExpressionNames.scala   |  1 +
 3 files changed, 18 insertions(+)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index c7dafdd480..ad17b38196 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -774,6 +774,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
       Sig[CollectSet](ExpressionNames.COLLECT_SET),
       Sig[VeloxBloomFilterMightContain](ExpressionNames.MIGHT_CONTAIN),
       Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG),
+      Sig[MapFilter](ExpressionNames.MAP_FILTER),
       // For test purpose.
       Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION)
     )
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 5a4af560b2..34c90e9077 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -614,6 +614,22 @@ abstract class ScalarFunctionsValidateSuite extends 
FunctionsValidateSuite {
     }
   }
 
+  test("map_filter") {
+    withTempPath {
+      path =>
+        Seq((Map("a" -> 1, "b" -> 2, "c" -> 3)))
+          .toDF("m")
+          .write
+          .parquet(path.getCanonicalPath)
+
+        
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl")
+
+        runQueryAndCompare("select map_filter(m, (k, v) -> k != 'b') from 
map_tbl") {
+          checkGlutenOperatorMatch[ProjectExecTransformer]
+        }
+    }
+  }
+
   test("test transform_keys function") {
     withTempPath {
       path =>
diff --git 
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
 
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 2c919cebe9..4ea3227b45 100644
--- 
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++ 
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -289,6 +289,7 @@ object ExpressionNames {
   final val TRANSFORM_KEYS = "transform_keys"
   final val TRANSFORM_VALUES = "transform_values"
   final val STR_TO_MAP = "str_to_map"
+  final val MAP_FILTER = "map_filter"
 
   // struct functions
   final val GET_STRUCT_FIELD = "get_struct_field"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org