This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new e703d2f [SPARK-60051] Fix `DataFrame.collect()` to return `Data` for
`BINARY` columns instead of crashing
e703d2f is described below
commit e703d2f8aa3685565970009238a8ac4fb1678857
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Oct 7 13:11:52 2026 -0700
[SPARK-60051] Fix `DataFrame.collect()` to return `Data` for `BINARY`
columns instead of crashing
### What changes were proposed in this pull request?
This PR fixes `DataFrame.collect()` to read `BINARY` column values directly
from `BinaryArray` as `Data`, instead of going through `BinaryArray.asString`
and returning a `String.UTF8View`. `Row` equality now also compares `Data`
values.
### Why are the changes needed?
`BinaryArray.asString` force-unwraps `String(data:encoding: .utf8)`, so
collecting any binary value that is not valid UTF-8 (e.g. `X'FF00FE'`) crashes
the process. Returning `Data` also matches PySpark, which returns `bytearray`.
### Does this PR introduce _any_ user-facing change?
Yes. `BINARY` values in collected rows are now `Data` instead of
`String.UTF8View`, and non-UTF-8 binary values no longer crash.
### How was this patch tested?
Pass the CIs with the newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Sonnet 5.5
Closes #583 from dongjoon-hyun/claude/fix-binary-collect.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/DataFrame+Actions.swift | 4 ++--
Sources/SparkConnect/Row.swift | 2 ++
Tests/SparkConnectTests/CreateDataFrameTests.swift | 1 +
Tests/SparkConnectTests/DataFrameTests.swift | 9 +++++++++
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/Sources/SparkConnect/DataFrame+Actions.swift
b/Sources/SparkConnect/DataFrame+Actions.swift
index 414b237..835d1d3 100644
--- a/Sources/SparkConnect/DataFrame+Actions.swift
+++ b/Sources/SparkConnect/DataFrame+Actions.swift
@@ -172,11 +172,11 @@ extension DataFrame {
values.append(TimestampNanos(epochNanos: timestamp))
}
case ArrowType.ArrowBinary:
- guard let binaryArray = array as? AsString else {
+ guard let binaryArray = array as? BinaryArray else {
throw SparkConnectError.invalidArrowData(
SparkConnectError.Details(message: "Invalid Arrow `binary`
column."))
}
- values.append(binaryArray.asString(i).utf8)
+ values.append(binaryArray[i])
case .complexInfo(.strct):
guard let structArray = array as? AsString else {
throw SparkConnectError.invalidArrowData(
diff --git a/Sources/SparkConnect/Row.swift b/Sources/SparkConnect/Row.swift
index 35f6ad1..61db2bf 100644
--- a/Sources/SparkConnect/Row.swift
+++ b/Sources/SparkConnect/Row.swift
@@ -435,6 +435,8 @@ public struct Row: Sendable, Equatable {
return a == b
} else if let a = x as? String, let b = y as? String {
return a == b
+ } else if let a = x as? Data, let b = y as? Data {
+ return a == b
} else if let a = x as? [Bool], let b = y as? [Bool] {
return a == b
} else if let a = x as? [any FixedWidthInteger], let b = y as? [any
FixedWidthInteger] {
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift
b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index 15f73c5..55b0a29 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -151,6 +151,7 @@ struct CreateDataFrameTests {
let spark = try await SparkSession.builder.getOrCreate()
let df = try await spark.createDataFrame([[Data([1, 2, 3])], [nil]], "a
BINARY")
#expect(try await df.count() == 2)
+ #expect(try await df.collect() == [Row(Data([1, 2, 3])), Row(nil)])
await spark.stop()
}
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index 64a4efc..5889470 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -380,6 +380,15 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func collectBinary() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.sql("SELECT X'FF00FE' AS b, CAST(NULL AS
BINARY) AS n").collect()
+ #expect(rows == [Row(Data([0xFF, 0x00, 0xFE]), nil)])
+ #expect(try rows[0].get(0) as? Data == Data([0xFF, 0x00, 0xFE]))
+ await spark.stop()
+ }
+
@Test
func selectNone() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
