This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5b0a667d9a5c [SPARK-54522][SQL][TEST] Variant test suite fixes for
shredding configs
5b0a667d9a5c is described below
commit 5b0a667d9a5cf1f7fe1107844807110dec5fc553
Author: Harsh Motwani <[email protected]>
AuthorDate: Thu Nov 27 16:45:18 2025 -0800
[SPARK-54522][SQL][TEST] Variant test suite fixes for shredding configs
### What changes were proposed in this pull request?
This PR makes minor changes to test suites so that they pass with variant
shredding-related configs enabled, without hurting the purpose of the tests.
### Why are the changes needed?
These changes would help tests pass if shredding configs were enabled by
default in the future.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53242 from harshmotw-db/harshmotw-db/suite_fixes.
Authored-by: Harsh Motwani <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala | 4 +++-
.../execution/datasources/parquet/ParquetVariantShreddingSuite.scala | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
index 16be9558409c..6510a165e258 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
@@ -820,7 +820,9 @@ class VariantSuite extends QueryTest with
SharedSparkSession with ExpressionEval
// The initial size of the buffer backing a cached dataframe column is
128KB.
// See `ColumnBuilder`.
val numKeys = 128 * 1024
- val keyIterator = (0 until numKeys).iterator
+ // We start in long range because the shredded writer writes int64 by
default which wouldn't
+ // match narrower binaries.
+ val keyIterator = (Int.MaxValue + 1L until Int.MaxValue + 1L +
numKeys).iterator
val entries = Array.fill(numKeys)(s"""\"${keyIterator.next()}\":
\"test\"""")
val jsonStr = s"{${entries.mkString(", ")}}"
val query = s"""select named_struct(
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
index 1f06ddb29bd4..1cc6d3afbee5 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
@@ -379,7 +379,8 @@ class ParquetVariantShreddingSuite extends QueryTest with
ParquetTest with Share
"struct<value binary, typed_value int>>>"
withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString,
SQLConf.VARIANT_ALLOW_READING_SHREDDED.key -> true.toString,
- SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) {
+ SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema,
+ SQLConf.PARQUET_IGNORE_VARIANT_ANNOTATION.key -> true.toString) {
df.write.mode("overwrite").parquet(dir.getAbsolutePath)
// Verify that we can read the full variant.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
