This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6e1b48ab36 [GLUTEN-10159][VL] Fix json tuple with dot-containing field
names (#10939)
6e1b48ab36 is described below
commit 6e1b48ab36db382f87024bb0ab6670104b81eeee
Author: liangyongyuan <[email protected]>
AuthorDate: Fri Nov 21 13:26:05 2025 +0800
[GLUTEN-10159][VL] Fix json tuple with dot-containing field names (#10939)
---
.../apache/gluten/execution/GenerateExecTransformer.scala | 12 ++++++------
.../org/apache/gluten/execution/MiscOperatorSuite.scala | 9 +++++++++
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
index 8cca495333..c4b2e84d67 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
@@ -178,7 +178,6 @@ object GenerateExecTransformer {
}
object PullOutGenerateProjectHelper extends PullOutProjectHelper {
- val JSON_PATH_PREFIX = "$."
def pullOutPreProject(generate: GenerateExec): SparkPlan = {
if (GenerateExecTransformer.supportsGenerate(generate.generator)) {
generate.generator match {
@@ -243,15 +242,16 @@ object PullOutGenerateProjectHelper extends
PullOutProjectHelper {
case jsonPath if jsonPath.foldable =>
Option(jsonPath.eval()) match {
case Some(path) =>
- GetJsonObject(jsonObj, Literal.create(JSON_PATH_PREFIX +
path))
+ GetJsonObject(jsonObj, Literal.create("$[" + path + "]"))
case _ =>
Literal.create(null)
}
case jsonPath =>
- // TODO: The prefix is just for adapting to GetJsonObject.
- // Maybe, we can remove this handling in the future by
- // making path without "$." recognized
- GetJsonObject(jsonObj,
Concat(Seq(Literal.create(JSON_PATH_PREFIX), jsonPath)))
+ // Build bracket notation uniformly to
+ // allow dot-containing field names in JSON paths, e.g., $[a.b]
+ GetJsonObject(
+ jsonObj,
+ Concat(Seq(Literal.create("$["), jsonPath,
Literal.create("]"))))
}.toIndexedSeq
}
val preGenerateExprs =
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 8903922e74..f67ec53ec0 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -1897,6 +1897,15 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
}
+ withTempView("json_tuple_test") {
+ Seq[(String)](("{\"a.b\":\"b\"}"))
+ .toDF("json_field")
+ .createOrReplaceTempView("json_tuple_test")
+ runQueryAndCompare("SELECT json_tuple(json_field, 'a.b') from
json_tuple_test") {
+ checkGlutenOperatorMatch[GenerateExecTransformer]
+ }
+ }
+
runQueryAndCompare(
"""
|SELECT
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
