This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new c74fdc9db6d3 [SPARK-59828][SQL] Support `skipFixedLenByteArray` in 
`VectorizedDeltaByteArrayReader`
c74fdc9db6d3 is described below

commit c74fdc9db6d3311ef25559c63e05a67098b7d83e
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Sep 28 11:10:31 2026 -0700

    [SPARK-59828][SQL] Support `skipFixedLenByteArray` in 
`VectorizedDeltaByteArrayReader`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to implement `skipFixedLenByteArray` in 
`VectorizedDeltaByteArrayReader` by delegating to `skipBinary`, because 
`DELTA_BYTE_ARRAY` encodes `FIXED_LEN_BYTE_ARRAY` values in the same way as 
`BYTE_ARRAY` values.
    
    ### Why are the changes needed?
    
    The Parquet v2 writer encodes `FIXED_LEN_BYTE_ARRAY` columns such as 
`DECIMAL(38, 18)` with `DELTA_BYTE_ARRAY` when dictionary encoding is disabled. 
When Parquet column index filtering skips rows of such a column, the vectorized 
reader fails because `VectorizedDeltaByteArrayReader` does not implement 
`skipFixedLenByteArray`.
    
    ```scala
    spark.range(0, 2000).selectExpr("id as _1", "cast(id as decimal(38, 18)) as 
_2")
      .coalesce(1)
      .write
      .option("parquet.writer.version", "PARQUET_2_0")
      .option("parquet.enable.dictionary", "false")
      .option("parquet.page.size", "2048")
      .option("parquet.page.row.count.limit", "500")
      .parquet("t")
    spark.read.parquet("t").filter("_1 >= 500 and _1 < 1000").collect()
    ```
    
    ```
    org.apache.spark.SparkUnsupportedOperationException: 
[UNSUPPORTED_CALL.WITHOUT_SUGGESTION] Cannot call the method 
"skipFixedLenByteArray" of the class 
"org.apache.spark.sql.execution.datasources.parquet.VectorizedReaderBase".  
SQLSTATE: 0A000
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this fixes a bug that has existed since Apache Spark 
[v3.3.0](https://github.com/apache/spark/releases/tag/v3.3.0) (2022-06-09) but 
it didn't work before.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test cases, which fail without this fix.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #59101 from dongjoon-hyun/SPARK-59828.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit 9e3396008af900b36978aeff8b4fc96c53fa6090)
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit d0ddbccefa7627f5a52cb84c8bc6a1ed8a4cf774)
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit e35db51c005abfacc0ad94a62c6bc7c8a044613b)
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../parquet/VectorizedDeltaByteArrayReader.java     |  6 ++++++
 .../parquet/ParquetColumnIndexSuite.scala           | 21 +++++++++++++++++++++
 .../ParquetDeltaByteArrayEncodingSuite.scala        | 15 +++++++++++++++
 3 files changed, 42 insertions(+)

diff --git 
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaByteArrayReader.java
 
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaByteArrayReader.java
index fe65e0c82ac8..77cb08a295d0 100644
--- 
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaByteArrayReader.java
+++ 
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaByteArrayReader.java
@@ -176,4 +176,10 @@ public class VectorizedDeltaByteArrayReader extends 
VectorizedReaderBase
     }
   }
 
+  @Override
+  public void skipFixedLenByteArray(int total, int len) {
+    // DELTA_BYTE_ARRAY encodes FIXED_LEN_BYTE_ARRAY values in the same way as 
BYTE_ARRAY values.
+    skipBinary(total);
+  }
+
 }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnIndexSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnIndexSuite.scala
index 4e103e771d16..8a5e96e266a5 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnIndexSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnIndexSuite.scala
@@ -115,4 +115,25 @@ class ParquetColumnIndexSuite extends QueryTest with 
ParquetTest with SharedSpar
       df => df.filter("(s._1 >= 500 and s._1 < 1000) or (s._1 >= 1500 and s._1 
< 1600)")
     )
   }
+
+  test("SPARK-59828: reading unaligned pages - FIXED_LEN_BYTE_ARRAY with 
DELTA_BYTE_ARRAY") {
+    // Without a dictionary, the v2 writer encodes FIXED_LEN_BYTE_ARRAY as 
DELTA_BYTE_ARRAY.
+    // The pages of _2 are split by size, and those of _1 by row count because 
its
+    // DELTA_BINARY_PACKED values are tiny, so that the pages of the two 
columns are unaligned.
+    val df = spark.range(0, 2000).selectExpr("id as _1", "cast(id as 
decimal(38, 18)) as _2")
+    withTempPath { file =>
+      df.coalesce(1)
+        .write
+        .option("parquet.writer.version", "PARQUET_2_0")
+        .option("parquet.enable.dictionary", "false")
+        .option("parquet.page.size", "2048")
+        .option("parquet.page.row.count.limit", "500")
+        .parquet(file.getCanonicalPath)
+
+      val parquetDf = spark.read.parquet(file.getCanonicalPath)
+      actions.foreach { action =>
+        checkAnswer(action(parquetDf), action(df))
+      }
+    }
+  }
 }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDeltaByteArrayEncodingSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDeltaByteArrayEncodingSuite.scala
index 1fbe51eea3b7..4db6dd633bf7 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDeltaByteArrayEncodingSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDeltaByteArrayEncodingSuite.scala
@@ -65,6 +65,21 @@ class ParquetDeltaByteArrayEncodingSuite extends 
ParquetCompatibilityTest with S
     assertReadWriteWithSkipN(writer, reader, randvalues)
   }
 
+  test("SPARK-59828: fixed-length byte arrays with skipN") {
+    // FIXED_LEN_BYTE_ARRAY values, e.g. decimal(38, 18), can also be 
DELTA_BYTE_ARRAY encoded.
+    val len = 16
+    val vals = Array.tabulate(1000)(i => f"$i%016d")
+    Utils.writeData(writer, vals)
+    reader.initFromPage(vals.length, writer.getBytes.toInputStream)
+    var i = 0
+    while (i < vals.length) {
+      val skipCount = (vals.length - i) / 2
+      assert(vals(i).getBytes() sameElements reader.readBinary(len).getBytes)
+      reader.skipFixedLenByteArray(skipCount, len)
+      i += skipCount + 1
+    }
+  }
+
   test("prefix length larger than the previous value is rejected") {
     // Craft a page by hand: a benign 2-byte first value, then a value whose 
prefix length
     // claims 65536 bytes of a 2-byte previous value.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to