This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new ca8818b9bd58 [SPARK-58510][SQL] Make variant path functions error
consistently on malformed constant paths across execution modes
ca8818b9bd58 is described below
commit ca8818b9bd582cb27dd51883d5908611f4922e66
Author: bojana-db <[email protected]>
AuthorDate: Mon Aug 3 09:50:22 2026 +0000
[SPARK-58510][SQL] Make variant path functions error consistently on
malformed constant paths across execution modes
### What changes were proposed in this pull request?
Override `eval` in `VariantGet`, `VariantInsert`, and `VariantArrayAppend`
to force parsing of a foldable (constant) JSONPath before the null-intolerant
short-circuit, so a malformed constant path fails the same way in interpreted
and codegen execution.
### Why are the changes needed?
Codegen parses a foldable path at code-generation time (always throwing on
a malformed path), while interpreted execution only parses inside
`nullSafeEval`, which is skipped when any argument is `NULL`. So a malformed
constant path with a `NULL` argument returned `NULL` in interpreted mode but
threw in codegen..
### Does this PR introduce any user-facing change?
In interpreted mode, `variant_get`/`variant_insert`/`variant_array_append`
(and their `try_*` variants) with a malformed constant path and a `NULL`
argument now raise `INVALID_VARIANT_PATH`/`INVALID_VARIANT_GET_PATH` instead of
returning `NULL`, matching codegen.
### How was this patch tested?
Added unit tests in `VariantExpressionSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code with Claude Opus 4.8
Closes #57682 from bojana-db/fix-foldable-path-error.
Authored-by: bojana-db <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
(cherry picked from commit bf57ee78058165ffb994ac8462d768f61f2f7a9a)
Signed-off-by: Uros Bojanic <[email protected]>
---
.../expressions/variant/variantExpressions.scala | 15 +++++++++++
.../variant/VariantExpressionSuite.scala | 30 ++++++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index e9ce291b655c..3448bfee62e1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -289,6 +289,11 @@ case class VariantGet(
timeZoneId,
zoneId)
+ override def eval(input: InternalRow): Any = {
+ val _ = parsedPath
+ super.eval(input)
+ }
+
protected override def nullSafeEval(input: Any, path: Any): Any = parsedPath
match {
case Some(pp) =>
VariantGet.variantGet(input.asInstanceOf[VariantVal], pp, dataType,
castArgs)
@@ -873,6 +878,11 @@ case class VariantInsert(
}
}
+ override def eval(input: InternalRow): Any = {
+ val _ = foldablePath
+ super.eval(input)
+ }
+
override protected def nullSafeEval(v: Any, p: Any, valValue: Any): Any = {
val inputVariant = v.asInstanceOf[VariantVal]
foldablePath match {
@@ -1308,6 +1318,11 @@ case class VariantArrayAppend(
}
}
+ override def eval(input: InternalRow): Any = {
+ val _ = foldablePath
+ super.eval(input)
+ }
+
override protected def nullSafeEval(v: Any, p: Any, valValue: Any): Any = {
val inputVariant = v.asInstanceOf[VariantVal]
foldablePath match {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index eaf9ac267ff0..83a0aaa2e6a6 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -686,6 +686,16 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
// scalastyle:on nonascii
testVariantGet("[1, 2, 3]", "$[2147483647]", IntegerType, null)
+ Seq("variant_get" -> true, "try_variant_get" -> false).foreach {
+ case (name, failOnError) =>
+ checkErrorInExpression[SparkRuntimeException](
+ VariantGet(BoundReference(0, VariantType, nullable = true),
Literal(".a"),
+ IntegerType, failOnError),
+ InternalRow(null),
+ "INVALID_VARIANT_GET_PATH",
+ Map("path" -> ".a", "functionName" -> s"`$name`"))
+ }
+
checkInvalidPath("")
checkInvalidPath(".a")
checkInvalidPath("$1")
@@ -1487,6 +1497,16 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
"INVALID_VARIANT_PATH",
name => Map("path" -> "abc", "functionName" -> s"`$name`"))
+ Seq("variant_insert" -> true, "try_variant_insert" -> false).foreach {
+ case (name, failOnError) =>
+ checkErrorInExpression[SparkRuntimeException](
+ VariantInsert(BoundReference(0, VariantType, nullable = true),
+ Literal.create("abc", StringType), Literal(1), failOnError),
+ InternalRow(null),
+ "INVALID_VARIANT_PATH",
+ Map("path" -> "abc", "functionName" -> s"`$name`"))
+ }
+
val tooBig = "x".repeat(16 * 1024 * 1024)
checkInsertUnrecoverableError("{}", "$.a[2000000000]", Literal(1),
"VARIANT_SIZE_LIMIT",
@@ -1840,6 +1860,16 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkAppendUnrecoverableError("[]", "abc", Literal(1),
"INVALID_VARIANT_PATH",
name => Map("path" -> "abc", "functionName" -> s"`$name`"))
+
+ Seq("variant_array_append" -> true, "try_variant_array_append" ->
false).foreach {
+ case (name, failOnError) =>
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(BoundReference(0, VariantType, nullable = true),
+ Literal.create("abc", StringType), Literal(1), failOnError),
+ InternalRow(null),
+ "INVALID_VARIANT_PATH",
+ Map("path" -> "abc", "functionName" -> s"`$name`"))
+ }
checkErrorInExpression[SparkRuntimeException](
VariantArrayAppend(Literal(parseJson("[]")), Literal(""), Literal(1)),
"INVALID_VARIANT_PATH",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]