This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new ca8818b9bd58 [SPARK-58510][SQL] Make variant path functions error 
consistently on malformed constant paths across execution modes
ca8818b9bd58 is described below

commit ca8818b9bd582cb27dd51883d5908611f4922e66
Author: bojana-db <[email protected]>
AuthorDate: Mon Aug 3 09:50:22 2026 +0000

    [SPARK-58510][SQL] Make variant path functions error consistently on 
malformed constant paths across execution modes
    
    ### What changes were proposed in this pull request?
    Override `eval` in `VariantGet`, `VariantInsert`, and `VariantArrayAppend` 
to force parsing of a foldable (constant) JSONPath before the null-intolerant 
short-circuit, so a malformed constant path fails the same way in interpreted 
and codegen execution.
    
    ### Why are the changes needed?
    Codegen parses a foldable path at code-generation time (always throwing on 
a malformed path), while interpreted execution only parses inside 
`nullSafeEval`, which is skipped when any argument is `NULL`. So a malformed 
constant path with a `NULL` argument returned `NULL` in interpreted mode but 
threw in codegen..
    
    ### Does this PR introduce any user-facing change?
    In interpreted mode, `variant_get`/`variant_insert`/`variant_array_append` 
(and their `try_*` variants) with a malformed constant path and a `NULL` 
argument now raise `INVALID_VARIANT_PATH`/`INVALID_VARIANT_GET_PATH` instead of 
returning `NULL`, matching codegen.
    
    ### How was this patch tested?
    Added unit tests in `VariantExpressionSuite`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Generated-by: Claude Code with Claude Opus 4.8
    
    Closes #57682 from bojana-db/fix-foldable-path-error.
    
    Authored-by: bojana-db <[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
    (cherry picked from commit bf57ee78058165ffb994ac8462d768f61f2f7a9a)
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .../expressions/variant/variantExpressions.scala   | 15 +++++++++++
 .../variant/VariantExpressionSuite.scala           | 30 ++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index e9ce291b655c..3448bfee62e1 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -289,6 +289,11 @@ case class VariantGet(
     timeZoneId,
     zoneId)
 
+  override def eval(input: InternalRow): Any = {
+    val _ = parsedPath
+    super.eval(input)
+  }
+
   protected override def nullSafeEval(input: Any, path: Any): Any = parsedPath 
match {
     case Some(pp) =>
       VariantGet.variantGet(input.asInstanceOf[VariantVal], pp, dataType, 
castArgs)
@@ -873,6 +878,11 @@ case class VariantInsert(
     }
   }
 
+  override def eval(input: InternalRow): Any = {
+    val _ = foldablePath
+    super.eval(input)
+  }
+
   override protected def nullSafeEval(v: Any, p: Any, valValue: Any): Any = {
     val inputVariant = v.asInstanceOf[VariantVal]
     foldablePath match {
@@ -1308,6 +1318,11 @@ case class VariantArrayAppend(
     }
   }
 
+  override def eval(input: InternalRow): Any = {
+    val _ = foldablePath
+    super.eval(input)
+  }
+
   override protected def nullSafeEval(v: Any, p: Any, valValue: Any): Any = {
     val inputVariant = v.asInstanceOf[VariantVal]
     foldablePath match {
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index eaf9ac267ff0..83a0aaa2e6a6 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -686,6 +686,16 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     // scalastyle:on nonascii
     testVariantGet("[1, 2, 3]", "$[2147483647]", IntegerType, null)
 
+    Seq("variant_get" -> true, "try_variant_get" -> false).foreach {
+      case (name, failOnError) =>
+        checkErrorInExpression[SparkRuntimeException](
+          VariantGet(BoundReference(0, VariantType, nullable = true), 
Literal(".a"),
+            IntegerType, failOnError),
+          InternalRow(null),
+          "INVALID_VARIANT_GET_PATH",
+          Map("path" -> ".a", "functionName" -> s"`$name`"))
+    }
+
     checkInvalidPath("")
     checkInvalidPath(".a")
     checkInvalidPath("$1")
@@ -1487,6 +1497,16 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
       "INVALID_VARIANT_PATH",
       name => Map("path" -> "abc", "functionName" -> s"`$name`"))
 
+    Seq("variant_insert" -> true, "try_variant_insert" -> false).foreach {
+      case (name, failOnError) =>
+        checkErrorInExpression[SparkRuntimeException](
+          VariantInsert(BoundReference(0, VariantType, nullable = true),
+            Literal.create("abc", StringType), Literal(1), failOnError),
+          InternalRow(null),
+          "INVALID_VARIANT_PATH",
+          Map("path" -> "abc", "functionName" -> s"`$name`"))
+    }
+
     val tooBig = "x".repeat(16 * 1024 * 1024)
     checkInsertUnrecoverableError("{}", "$.a[2000000000]", Literal(1),
       "VARIANT_SIZE_LIMIT",
@@ -1840,6 +1860,16 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     checkAppendUnrecoverableError("[]", "abc", Literal(1),
       "INVALID_VARIANT_PATH",
       name => Map("path" -> "abc", "functionName" -> s"`$name`"))
+
+    Seq("variant_array_append" -> true, "try_variant_array_append" -> 
false).foreach {
+      case (name, failOnError) =>
+        checkErrorInExpression[SparkRuntimeException](
+          VariantArrayAppend(BoundReference(0, VariantType, nullable = true),
+            Literal.create("abc", StringType), Literal(1), failOnError),
+          InternalRow(null),
+          "INVALID_VARIANT_PATH",
+          Map("path" -> "abc", "functionName" -> s"`$name`"))
+    }
     checkErrorInExpression[SparkRuntimeException](
       VariantArrayAppend(Literal(parseJson("[]")), Literal(""), Literal(1)),
       "INVALID_VARIANT_PATH",


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to