This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 8283f29a7c2e [SPARK-60713][SQL] Report the actual size limit in 
`VARIANT_CONSTRUCTOR_SIZE_LIMIT`
8283f29a7c2e is described below

commit 8283f29a7c2e85ee5b8503058322a3e11bd370fc
Author: bajka <[email protected]>
AuthorDate: Thu Nov 5 14:56:24 2026 +0000

    [SPARK-60713][SQL] Report the actual size limit in 
`VARIANT_CONSTRUCTOR_SIZE_LIMIT`
    
    ### What changes were proposed in this pull request?
    
    This PR replaces the hardcoded 16 MiB with a <sizeLimit>  parameter, which 
`VariantUtil.variantConstructorSizeLimit()` fills in from 
`VariantUtil.SIZE_LIMIT`. The enforced limit and the error class are unchanged.
    
    ### Why are the changes needed?
    
    SPARK-52181 raised the Variant size limit from 16 MiB to 128MiB, but this 
error message still said 16 MiB. A Variant larger than 128 MiB was rejected 
with a message saying the limit is 16 MiB.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, the error message text.
    - Before: `Cannot construct a Variant larger than 16 MiB. The maximum  
allowed size of a Variant value is 16 MiB.`
    - After: `Cannot construct a Variant larger than 128 MiB. The maximum 
allowed size of a Variant value is 128 MiB.`
    
    ### How was this patch tested?
    
    Using updated tests in `VariantExpressionSuite` ("to_json malformed") to 
check the `sizeLimit` parameter of `VARIANT_CONSTRUCTOR_SIZE_LIMIT`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 5.5)
    
    Closes #59928 from SavicStefan/effort/variant-size-limit.
    
    Lead-authored-by: bajka <[email protected]>
    Co-authored-by: Stefan Savić <[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .../utils/src/main/resources/error/error-conditions.json   |  2 +-
 .../main/java/org/apache/spark/types/variant/Variant.java  |  4 ++--
 .../java/org/apache/spark/types/variant/VariantUtil.java   | 11 +++++++----
 .../expressions/variant/VariantExpressionSuite.scala       | 14 +++++++++-----
 4 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/common/utils/src/main/resources/error/error-conditions.json 
b/common/utils/src/main/resources/error/error-conditions.json
index 0cb7bc927152..7b62bf3db43c 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -10308,7 +10308,7 @@
   },
   "VARIANT_CONSTRUCTOR_SIZE_LIMIT" : {
     "message" : [
-      "Cannot construct a Variant larger than 16 MiB. The maximum allowed size 
of a Variant value is 16 MiB."
+      "Cannot construct a Variant larger than <sizeLimit>. The maximum allowed 
size of a Variant value is <sizeLimit>."
     ],
     "sqlState" : "22023"
   },
diff --git 
a/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java 
b/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
index cefb957f9b97..4f600b2c65c0 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
@@ -60,8 +60,8 @@ public final class Variant {
     if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
       throw malformedVariant();
     }
-    // Don't attempt to use a Variant larger than 16 MiB. We'll never produce 
one, and it risks
-    // memory instability.
+    // Don't attempt to use a Variant larger than `SIZE_LIMIT`. We'll never 
produce one, and it
+    // risks memory instability.
     if (metadata.length > SIZE_LIMIT || value.length > SIZE_LIMIT) {
       throw variantConstructorSizeLimit();
     }
diff --git 
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java 
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 64044ad49eb7..26be937a89e9 100644
--- 
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++ 
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -19,6 +19,7 @@ package org.apache.spark.types.variant;
 
 import org.apache.spark.QueryContext;
 import org.apache.spark.SparkRuntimeException;
+import org.apache.spark.network.util.ByteUnit;
 import org.apache.spark.network.util.JavaUtils;
 import scala.collection.immutable.Map$;
 
@@ -196,12 +197,14 @@ public class VariantUtil {
   }
 
   // An exception indicating that an external caller tried to call the Variant 
constructor with
-  // value or metadata exceeding the 16MiB size limit. We will never construct 
a Variant this large,
-  // so it should only be possible to encounter this exception when reading a 
Variant produced by
-  // another tool.
+  // value or metadata exceeding `SIZE_LIMIT`. We will never construct a 
Variant this large, so it
+  // should only be possible to encounter this exception when reading a 
Variant produced by another
+  // tool.
   static SparkRuntimeException variantConstructorSizeLimit() {
     return new SparkRuntimeException("VARIANT_CONSTRUCTOR_SIZE_LIMIT",
-        Map$.MODULE$.<String, String>empty(), null, new QueryContext[]{}, "");
+        new scala.collection.immutable.Map.Map1<>("sizeLimit",
+            ByteUnit.BYTE.toMiB(SIZE_LIMIT) + " MiB"),
+        null, new QueryContext[]{}, "");
   }
 
   // Check the validity of an array index `pos`. Throw `MALFORMED_VARIANT` if 
it is out of bound,
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index e23857541b14..bdee7820529a 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -51,11 +51,13 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     def check(
         value: Array[Byte],
         metadata: Array[Byte],
-        condition: String = "MALFORMED_VARIANT"): Unit = {
+        condition: String = "MALFORMED_VARIANT",
+        parameters: Map[String, String] = Map.empty): Unit = {
       checkErrorInExpression[SparkRuntimeException](
         ResolveTimeZone.resolveTimeZones(
           StructsToJson(Map.empty, Literal(new VariantVal(value, metadata)))),
-        condition
+        condition,
+        parameters
       )
     }
 
@@ -100,7 +102,9 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     val a = createArray[Byte](SIZE_LIMIT, 'a'.toByte)
     val hugeMetadata = bigVersion ++ Array[Byte](2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 1, 1, 0, 0, 1) ++
       a ++ Array[Byte]('b')
-    check(Array(primitiveHeader(TRUE)), hugeMetadata, 
"VARIANT_CONSTRUCTOR_SIZE_LIMIT")
+    val sizeLimitParams = Map("sizeLimit" -> "16 MiB")
+    check(Array(primitiveHeader(TRUE)), hugeMetadata, 
"VARIANT_CONSTRUCTOR_SIZE_LIMIT",
+      sizeLimitParams)
 
     // The keys are 'aaa....' and 'b'. Values are "yyy..." and 'true'.
     val y = createArray[Byte](SIZE_LIMIT, 'y'.toByte)
@@ -116,8 +120,8 @@ class VariantExpressionSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 
     val smallMetadata = bigVersion ++ Array[Byte](2, 0, 0, 0, 0, 0, 0, 0, 1, 
0, 0, 0, 2, 0, 0, 0) ++
       Array[Byte]('a', 'b')
-    check(hugeObject, smallMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT")
-    check(hugeObject, hugeMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT")
+    check(hugeObject, smallMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT", 
sizeLimitParams)
+    check(hugeObject, hugeMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT", 
sizeLimitParams)
   }
 
   // Test valid forms of Variant that our writer would never produce.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to