This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8283f29a7c2e [SPARK-60713][SQL] Report the actual size limit in
`VARIANT_CONSTRUCTOR_SIZE_LIMIT`
8283f29a7c2e is described below
commit 8283f29a7c2e85ee5b8503058322a3e11bd370fc
Author: bajka <[email protected]>
AuthorDate: Thu Nov 5 14:56:24 2026 +0000
[SPARK-60713][SQL] Report the actual size limit in
`VARIANT_CONSTRUCTOR_SIZE_LIMIT`
### What changes were proposed in this pull request?
This PR replaces the hardcoded 16 MiB with a <sizeLimit> parameter, which
`VariantUtil.variantConstructorSizeLimit()` fills in from
`VariantUtil.SIZE_LIMIT`. The enforced limit and the error class are unchanged.
### Why are the changes needed?
SPARK-52181 raised the Variant size limit from 16 MiB to 128MiB, but this
error message still said 16 MiB. A Variant larger than 128 MiB was rejected
with a message saying the limit is 16 MiB.
### Does this PR introduce _any_ user-facing change?
Yes, the error message text.
- Before: `Cannot construct a Variant larger than 16 MiB. The maximum
allowed size of a Variant value is 16 MiB.`
- After: `Cannot construct a Variant larger than 128 MiB. The maximum
allowed size of a Variant value is 128 MiB.`
### How was this patch tested?
Using updated tests in `VariantExpressionSuite` ("to_json malformed") to
check the `sizeLimit` parameter of `VARIANT_CONSTRUCTOR_SIZE_LIMIT`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5.5)
Closes #59928 from SavicStefan/effort/variant-size-limit.
Lead-authored-by: bajka <[email protected]>
Co-authored-by: Stefan Savić <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../utils/src/main/resources/error/error-conditions.json | 2 +-
.../main/java/org/apache/spark/types/variant/Variant.java | 4 ++--
.../java/org/apache/spark/types/variant/VariantUtil.java | 11 +++++++----
.../expressions/variant/VariantExpressionSuite.scala | 14 +++++++++-----
4 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index 0cb7bc927152..7b62bf3db43c 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -10308,7 +10308,7 @@
},
"VARIANT_CONSTRUCTOR_SIZE_LIMIT" : {
"message" : [
- "Cannot construct a Variant larger than 16 MiB. The maximum allowed size
of a Variant value is 16 MiB."
+ "Cannot construct a Variant larger than <sizeLimit>. The maximum allowed
size of a Variant value is <sizeLimit>."
],
"sqlState" : "22023"
},
diff --git
a/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
b/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
index cefb957f9b97..4f600b2c65c0 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/Variant.java
@@ -60,8 +60,8 @@ public final class Variant {
if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
throw malformedVariant();
}
- // Don't attempt to use a Variant larger than 16 MiB. We'll never produce
one, and it risks
- // memory instability.
+ // Don't attempt to use a Variant larger than `SIZE_LIMIT`. We'll never
produce one, and it
+ // risks memory instability.
if (metadata.length > SIZE_LIMIT || value.length > SIZE_LIMIT) {
throw variantConstructorSizeLimit();
}
diff --git
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 64044ad49eb7..26be937a89e9 100644
---
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -19,6 +19,7 @@ package org.apache.spark.types.variant;
import org.apache.spark.QueryContext;
import org.apache.spark.SparkRuntimeException;
+import org.apache.spark.network.util.ByteUnit;
import org.apache.spark.network.util.JavaUtils;
import scala.collection.immutable.Map$;
@@ -196,12 +197,14 @@ public class VariantUtil {
}
// An exception indicating that an external caller tried to call the Variant
constructor with
- // value or metadata exceeding the 16MiB size limit. We will never construct
a Variant this large,
- // so it should only be possible to encounter this exception when reading a
Variant produced by
- // another tool.
+ // value or metadata exceeding `SIZE_LIMIT`. We will never construct a
Variant this large, so it
+ // should only be possible to encounter this exception when reading a
Variant produced by another
+ // tool.
static SparkRuntimeException variantConstructorSizeLimit() {
return new SparkRuntimeException("VARIANT_CONSTRUCTOR_SIZE_LIMIT",
- Map$.MODULE$.<String, String>empty(), null, new QueryContext[]{}, "");
+ new scala.collection.immutable.Map.Map1<>("sizeLimit",
+ ByteUnit.BYTE.toMiB(SIZE_LIMIT) + " MiB"),
+ null, new QueryContext[]{}, "");
}
// Check the validity of an array index `pos`. Throw `MALFORMED_VARIANT` if
it is out of bound,
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index e23857541b14..bdee7820529a 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -51,11 +51,13 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
def check(
value: Array[Byte],
metadata: Array[Byte],
- condition: String = "MALFORMED_VARIANT"): Unit = {
+ condition: String = "MALFORMED_VARIANT",
+ parameters: Map[String, String] = Map.empty): Unit = {
checkErrorInExpression[SparkRuntimeException](
ResolveTimeZone.resolveTimeZones(
StructsToJson(Map.empty, Literal(new VariantVal(value, metadata)))),
- condition
+ condition,
+ parameters
)
}
@@ -100,7 +102,9 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
val a = createArray[Byte](SIZE_LIMIT, 'a'.toByte)
val hugeMetadata = bigVersion ++ Array[Byte](2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 0, 0, 1) ++
a ++ Array[Byte]('b')
- check(Array(primitiveHeader(TRUE)), hugeMetadata,
"VARIANT_CONSTRUCTOR_SIZE_LIMIT")
+ val sizeLimitParams = Map("sizeLimit" -> "16 MiB")
+ check(Array(primitiveHeader(TRUE)), hugeMetadata,
"VARIANT_CONSTRUCTOR_SIZE_LIMIT",
+ sizeLimitParams)
// The keys are 'aaa....' and 'b'. Values are "yyy..." and 'true'.
val y = createArray[Byte](SIZE_LIMIT, 'y'.toByte)
@@ -116,8 +120,8 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
val smallMetadata = bigVersion ++ Array[Byte](2, 0, 0, 0, 0, 0, 0, 0, 1,
0, 0, 0, 2, 0, 0, 0) ++
Array[Byte]('a', 'b')
- check(hugeObject, smallMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT")
- check(hugeObject, hugeMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT")
+ check(hugeObject, smallMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT",
sizeLimitParams)
+ check(hugeObject, hugeMetadata, "VARIANT_CONSTRUCTOR_SIZE_LIMIT",
sizeLimitParams)
}
// Test valid forms of Variant that our writer would never produce.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]