This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new c579369  [SPARK-58526] Fix `VariableBufferBuilder` to handle large 
first value and offsets truncation
c579369 is described below

commit c579369b76cdcd870c5083e0519993959335793d
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Aug 3 14:00:31 2026 -0700

    [SPARK-58526] Fix `VariableBufferBuilder` to handle large first value and 
offsets truncation
    
    ### What changes were proposed in this pull request?
    
    This PR aims to fix two bugs in the vendored Arrow `VariableBufferBuilder` 
(used for `STRING`/`BINARY` serialization in `SparkSession.createDataFrame`).
    
    1. **Heap buffer overflow in `append()`**: the values-buffer capacity check 
was inside the `if index > 0` branch, so the very first value was written 
without any capacity check. When the first string/binary value is longer than 
the initial 64-byte buffer, the write overflows the heap allocation 
(intermittent segfaults or data corruption). The fix moves the capacity check 
out of the `if index > 0` branch so it applies to the first value as well.
    
    2. **Truncated offsets buffer in `finish()`**: the Arrow format requires 
`length + 1` offset entries because value `i` is read from the range 
`offsets[i]..<offsets[i + 1]`, but `finish()` allocated only `length` entries, 
dropping the final offset. The fix allocates `(length + 1) * 4` bytes for the 
offsets buffer while keeping its `.length` as `length`, because `ArrowData` 
uses the offsets buffer's `.length` as the array length.
    
    The same bugs exist in upstream `apache/arrow-swift`; they will be reported 
there separately.
    
    ### Why are the changes needed?
    
    Both bugs corrupt Arrow IPC data produced by `createDataFrame`:
    
    - Bug 1 causes intermittent crashes or corrupted values when the first 
string exceeds 64 bytes.
    - Bug 2 truncates the last offsets entry whenever `4 * rowCount` is a 
multiple of 64 (e.g., 16 rows), making the server fail with `Cannot grow 
BufferHolder by size <negative>`. In other cases, the 64-byte alignment padding 
happens to preserve the final offset, which is why the existing small-scale 
tests passed.
    
    Reproducer (fails or crashes before this fix, passes after):
    
    ```swift
    let spark = try await SparkSession.builder.getOrCreate()
    let value = String(repeating: "x", count: 200)
    let data: [[Sendable?]] = (0..<16).map { (i: Int) in [i, value + String(i)] 
}
    let rows = try await spark.createDataFrame(data, "id INT, value 
STRING").collect()
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this is a bug fix. `createDataFrame` now works correctly with 
string/binary data that previously crashed or was rejected by the server.
    
    ### How was this patch tested?
    
    Pass the CIs with a newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #471 from dongjoon-hyun/SPARK-58526.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/ArrowBufferBuilder.swift      | 15 ++++++++++-----
 Tests/SparkConnectTests/CreateDataFrameTests.swift | 14 ++++++++++++++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/Sources/SparkConnect/ArrowBufferBuilder.swift 
b/Sources/SparkConnect/ArrowBufferBuilder.swift
index 78bd53f..acffd59 100644
--- a/Sources/SparkConnect/ArrowBufferBuilder.swift
+++ b/Sources/SparkConnect/ArrowBufferBuilder.swift
@@ -239,9 +239,9 @@ public class VariableBufferBuilder<T>: 
ValuesBufferBuilder<T>, ArrowBufferBuilde
     if index > 0 {
       currentIndex = self.offsets.rawPointer.advanced(by: 
offsetIndex).load(as: Int32.self)
       currentOffset += currentIndex
-      if currentOffset > self.values.length {
-        self.value_resize(UInt(currentOffset))
-      }
+    }
+    if currentOffset > self.values.length {
+      self.value_resize(UInt(currentOffset))
     }
 
     if isNull {
@@ -287,10 +287,15 @@ public class VariableBufferBuilder<T>: 
ValuesBufferBuilder<T>, ArrowBufferBuilde
     let length = self.length
     var values = ArrowBuffer.createBuffer(self.values.length, size: 
UInt(MemoryLayout<UInt8>.size))
     var nulls = ArrowBuffer.createBuffer(length / 8 + 1, size: 
UInt(MemoryLayout<UInt8>.size))
-    var offsets = ArrowBuffer.createBuffer(length, size: 
UInt(MemoryLayout<Int32>.size))
+    // The offsets buffer requires `length + 1` entries because each value `i` 
is read from the
+    // range `offsets[i]..<offsets[i + 1]`, while its buffer length must 
remain `length` because
+    // it is used as the array length.
+    let offsetsByteCount = Int(length + 1) * MemoryLayout<Int32>.stride
+    let offsetsBytes = [UInt8](
+      UnsafeRawBufferPointer(start: self.offsets.rawPointer, count: 
offsetsByteCount))
+    let offsets = ArrowBuffer.createBuffer(offsetsBytes, length: length)
     ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
     ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
-    ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: offsets.capacity)
     return [nulls, offsets, values]
   }
 }
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift 
b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index b9a82d9..ef419de 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -125,4 +125,18 @@ struct CreateDataFrameTests {
     }
     await spark.stop()
   }
+
+  @Test
+  func longStrings() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    // A first string longer than the initial 64-byte buffer and a row count 
where
+    // `4 * rowCount` is a multiple of 64 in order to detect Arrow 
serialization regressions.
+    let value = String(repeating: "x", count: 200)
+    let data: [[Sendable?]] = (0..<16).map { (i: Int) in [i, value + 
String(i)] }
+    let rows = try await spark.createDataFrame(data, "id INT, value 
STRING").collect()
+    #expect(rows.count == 16)
+    #expect(try rows[0].get(1) as? String == value + "0")
+    #expect(try rows[15].get(1) as? String == value + "15")
+    await spark.stop()
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to