This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 192ff9a04d [VL] Offload str_to_map under the LAST_WIN map key dedup
policy (#13462)
192ff9a04d is described below
commit 192ff9a04df1c2aaa473c672cf77a816bddc0db4
Author: Anshul Baliga <[email protected]>
AuthorDate: Thu Oct 29 14:23:06 2026 +0530
[VL] Offload str_to_map under the LAST_WIN map key dedup policy (#13462)
---
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 19 -------------
.../gluten/expression/ExpressionRestrictions.scala | 13 ---------
.../functions/ScalarFunctionsValidateSuite.scala | 32 ++++++++++++++++++++++
3 files changed, 32 insertions(+), 32 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index a62fbf849c..4c64695507 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -1182,25 +1182,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi
with Logging {
}
}
- override def genStringToMapTransformer(
- substraitExprName: String,
- children: Seq[ExpressionTransformer],
- expr: Expression): ExpressionTransformer = {
- // Calling `.toString` on both sides ensures compatibility across all
Spark versions.
- // Starting from Spark 4.1,
`SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)` returns
- // an enum instead of a String.
- if (
- SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY).toString
- != SQLConf.MapKeyDedupPolicy.EXCEPTION.toString
- ) {
- GlutenExceptionUtil.throwsNotFullySupported(
- ExpressionNames.STR_TO_MAP,
- StrToMapRestrictions.ONLY_SUPPORT_MAP_KEY_DEDUP_POLICY
- )
- }
- GenericExpressionTransformer(substraitExprName, children, expr)
- }
-
override def genFormatNumberTransformer(
substraitExprName: String,
children: Seq[ExpressionTransformer],
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
index b6bfd4119b..24b1aa6f8e 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
@@ -23,18 +23,6 @@ trait ExpressionRestrictions {
val restrictionMessages: Array[String]
}
-object StrToMapRestrictions extends ExpressionRestrictions {
- val ONLY_SUPPORT_MAP_KEY_DEDUP_POLICY: String =
- s"Only ${SQLConf.MAP_KEY_DEDUP_POLICY.key} = " +
- s"${SQLConf.MapKeyDedupPolicy.EXCEPTION.toString} is supported for Velox
backend"
-
- override val functionName: String = ExpressionNames.STR_TO_MAP
-
- override val restrictionMessages: Array[String] = Array(
- ONLY_SUPPORT_MAP_KEY_DEDUP_POLICY
- )
-}
-
object FromJsonRestrictions extends ExpressionRestrictions {
val MUST_ENABLE_PARTIAL_RESULTS: String =
s"${ExpressionNames.FROM_JSON} with 'spark.sql.json.enablePartialResults =
false' " +
@@ -120,7 +108,6 @@ object ExpressionRestrictions {
// Called by gen-function-support-docs.py to get all restrictions.
def listAllRestrictions(): Array[ExpressionRestrictions] = {
Array(
- StrToMapRestrictions,
FromJsonRestrictions,
ToJsonRestrictions,
Unbase64Restrictions,
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index c454db85a2..73a630492d 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -1811,4 +1811,36 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
}
+
+ test("str_to_map with MAP_KEY_DEDUP_POLICY LAST_WIN") {
+ // Velox's str_to_map honors throw_exception_on_duplicate_map_keys, which
Gluten derives
+ // from spark.sql.mapKeyDedupPolicy, so LAST_WIN no longer needs to fall
back. Under
+ // LAST_WIN a repeated key keeps the position of its first occurrence and
takes the last
+ // value. Entries without a key/value delimiter carry a NULL value and
take part in
+ // deduplication as well.
+ withTempPath {
+ path =>
+ Seq("a:1,b:2,a:3", "x:10,x:20,x:30", "k,k", "k:1,k", "a:1,b:2,c:3")
+ .toDF("s")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("str_to_map_tbl")
+
+ withSQLConf(
+ SQLConf.MAP_KEY_DEDUP_POLICY.key ->
SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+ runQueryAndCompare("SELECT s, str_to_map(s, ',', ':') FROM
str_to_map_tbl") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT s, str_to_map(s) FROM str_to_map_tbl") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ // Pin the entry order, which map equality ignores.
+ runQueryAndCompare(
+ "SELECT s, map_keys(str_to_map(s, ',', ':')) FROM str_to_map_tbl")
{
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
