This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 6bf516156a51 [SPARK-50953][FOLLOW-UP] Allow whitespace/tab in
variantGet paths
6bf516156a51 is described below
commit 6bf516156a511efaeaf0c3f9167be0e062359c20
Author: Harsh Motwani <harsh.motw...@databricks.com>
AuthorDate: Sat Apr 5 21:39:32 2025 +0800
[SPARK-50953][FOLLOW-UP] Allow whitespace/tab in variantGet paths
### What changes were proposed in this pull request?
This PR allows for whitespaces to be used in variantGet paths.
### Why are the changes needed?
JSON keys can have whitespaces and there needs to be a way to extract them.
### Does this PR introduce _any_ user-facing change?
Yes, it allows for a more diverse set of variant_get paths.
### How was this patch tested?
Unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50484 from harshmotw-db/harsh-motwani_data/variant_get_path.
Authored-by: Harsh Motwani <harsh.motw...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
(cherry picked from commit ba7a537225a7891f3e0d650d4141ef7bb8d06368)
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../spark/sql/catalyst/expressions/variant/variantExpressions.scala | 2 ++
.../sql/catalyst/expressions/variant/VariantExpressionSuite.scala | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index 027acc64d73f..5831a29c00a1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -203,6 +203,8 @@ object VariantPathParser extends RegexParsers {
ArrayExtraction(index.toInt)
}
+ override def skipWhitespace: Boolean = false
+
// Parse key segment like `.name`, `['name']`, or `["name"]`.
private def key: Parser[VariantPathSegment] =
for {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index df816bd8165f..1cd8cc6228ef 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -455,7 +455,7 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
|"category":"fiction","reader":[{"age":25,"name":"bob"},{"age":26,"name":"jack"}],
|"price":22.99,"isbn":"0-395-19395-8"}],"bicycle":{"price":19.95,"color":"red"}},
|"email":"amy@only_for_json_udf_test.net","owner":"amy","zip
code":"94025",
- |"fb:testid":"1234","":"empty string","?":"Question Mark?"}
+ |"fb:testid":"1234","":"empty string","?":"Question Mark?", "
":"Whitespace", "\t": "Tab"}
|""".stripMargin
testVariantGet(json, "$.store.bicycle", StringType,
"""{"color":"red","price":19.95}""")
checkEvaluation(
@@ -472,6 +472,8 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
testVariantGet(json, "$[\"\"]", StringType, "empty string")
testVariantGet(json, "$['']", StringType, "empty string")
testVariantGet(json, "$[\"?\"]", StringType, "Question Mark?")
+ testVariantGet(json, "$[\" \"]", StringType, "Whitespace")
+ testVariantGet(json, "$[\"\t\"]", StringType, "Tab")
testVariantGet(json, "$['?']", StringType, "Question Mark?")
testVariantGet(
json,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
