This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new ab62a17 [SPARK-58497] Support `createDataFrame` in `SparkSession`
ab62a17 is described below
commit ab62a1766e540d283cc4d634c44a071e752025a5
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Aug 1 15:57:51 2026 -0700
[SPARK-58497] Support `createDataFrame` in `SparkSession`
### What changes were proposed in this pull request?
This PR aims to support `SparkSession.createDataFrame(_:_:)` to create a
`DataFrame` from local Swift data with a DDL-formatted schema.
```swift
let df = try await spark.createDataFrame(
[[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
```
The data is serialized into an `Apache Arrow` IPC stream via the in-tree
`ArrowWriter` and embedded into the plan as a `LocalRelation`. Supported types
are `BOOLEAN`, `TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`,
`STRING`, `BINARY`, `DATE`, and `TIMESTAMP`. `nil` is mapped to `NULL`. Data
larger than 128MiB throws the new `SparkConnectError.LocalRelationTooLarge`.
### Why are the changes needed?
Previously, there was no way to create a `DataFrame` from local Swift data.
This is one of the fundamental `SparkSession` APIs which all other Spark
Connect clients provide.
### Does this PR introduce _any_ user-facing change?
No. This is a new API addition.
### How was this patch tested?
Pass the CIs with the newly added `CreateDataFrameTests` test suite.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #468 from dongjoon-hyun/SPARK-58497.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/ConvertToArrow.swift | 129 +++++++++++++++++++++
Sources/SparkConnect/SparkConnectClient.swift | 12 ++
Sources/SparkConnect/SparkConnectError.swift | 1 +
Sources/SparkConnect/SparkSession.swift | 34 ++++++
Tests/SparkConnectTests/CreateDataFrameTests.swift | 128 ++++++++++++++++++++
5 files changed, 304 insertions(+)
diff --git a/Sources/SparkConnect/ConvertToArrow.swift
b/Sources/SparkConnect/ConvertToArrow.swift
new file mode 100644
index 0000000..6473282
--- /dev/null
+++ b/Sources/SparkConnect/ConvertToArrow.swift
@@ -0,0 +1,129 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+#if canImport(FoundationEssentials)
+import FoundationEssentials
+#else
+import Foundation
+#endif
+
+/// A utility to convert local data into an `Apache Arrow` IPC stream for
`LocalRelation`.
+enum ConvertToArrow {
+ /// Convert the given rows into an `Apache Arrow` IPC stream according to
the Spark schema.
+ /// - Parameters:
+ /// - data: An array of rows whose values are ordered like the schema
fields.
+ /// - schema: A ``StructType`` describing the field names and types.
+ /// - Returns: A serialized `Apache Arrow` IPC stream.
+ static func toArrowIPCStream(_ data: [[Sendable?]], _ schema: StructType)
throws -> Data {
+ for row in data where row.count != schema.fields.count {
+ throw SparkConnectError.InvalidArgument
+ }
+ let batchBuilder = RecordBatch.Builder()
+ for (i, field) in schema.fields.enumerated() {
+ let column = data.map { $0[i] }
+ let holder = try toArrowColumn(column, field.dataType)
+ batchBuilder.addColumn(
+ ArrowField(field.name, type: holder.type, isNullable: field.nullable),
arrowArray: holder)
+ }
+ switch batchBuilder.finish() {
+ case .success(let batch):
+ switch ArrowWriter().writeStreaming(
+ ArrowWriter.Info(.recordbatch, schema: batch.schema, batches: [batch]))
+ {
+ case .success(let ipcStream):
+ return ipcStream
+ case .failure:
+ throw SparkConnectError.InvalidArrowData
+ }
+ case .failure:
+ throw SparkConnectError.InvalidArrowData
+ }
+ }
+
+ /// Build an Arrow column from the given values according to the Spark data
type.
+ private static func toArrowColumn(_ column: [Sendable?], _ dataType:
DataType) throws
+ -> ArrowArrayHolder
+ {
+ switch dataType.kind {
+ case .boolean:
+ return try fill(ArrowArrayBuilders.loadBoolArrayBuilder(), column) { $0
as? Bool }
+ case .byte:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Int8>,
+ column) { toInt64($0).flatMap { Int8(exactly: $0) } }
+ case .short:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Int16>,
+ column) { toInt64($0).flatMap { Int16(exactly: $0) } }
+ case .integer:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Int32>,
+ column) { toInt64($0).flatMap { Int32(exactly: $0) } }
+ case .long:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Int64>,
+ column) { toInt64($0) }
+ case .float:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Float>,
+ column) { $0 as? Float }
+ case .double:
+ return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as
NumberArrayBuilder<Double>,
+ column) { ($0 as? Double) ?? ($0 as? Float).map(Double.init) }
+ case .string:
+ return try fill(ArrowArrayBuilders.loadStringArrayBuilder(), column) {
$0 as? String }
+ case .binary:
+ return try fill(ArrowArrayBuilders.loadBinaryArrayBuilder(), column) {
$0 as? Data }
+ case .date:
+ return try fill(ArrowArrayBuilders.loadDate32ArrayBuilder(), column) {
$0 as? Date }
+ case .timestamp:
+ return try fill(
+ ArrowArrayBuilders.loadTimestampArrayBuilder(.microseconds, timezone:
"UTC"), column
+ ) { ($0 as? Date).map { Int64(($0.timeIntervalSince1970 *
1_000_000).rounded()) } }
+ default:
+ throw SparkConnectError.InvalidType
+ }
+ }
+
+ /// Append the given values to the builder while mapping `nil` to `null`.
+ /// Unlike `appendAny`, a value which is not convertible throws
``SparkConnectError/InvalidType``
+ /// instead of being appended as `null` silently.
+ private static func fill<T, U>(
+ _ builder: ArrowArrayBuilder<T, U>, _ column: [Sendable?],
+ _ convert: (Sendable) -> T.ItemType?
+ ) throws -> ArrowArrayHolder {
+ for value in column {
+ if let value {
+ guard let converted = convert(value) else {
+ throw SparkConnectError.InvalidType
+ }
+ builder.append(converted)
+ } else {
+ builder.append(nil)
+ }
+ }
+ return try builder.toHolder()
+ }
+
+ private static func toInt64(_ value: Sendable) -> Int64? {
+ switch value {
+ case let v as Int: return Int64(v)
+ case let v as Int8: return Int64(v)
+ case let v as Int16: return Int64(v)
+ case let v as Int32: return Int64(v)
+ case let v as Int64: return v
+ default: return nil
+ }
+ }
+}
diff --git a/Sources/SparkConnect/SparkConnectClient.swift
b/Sources/SparkConnect/SparkConnectClient.swift
index e545766..aa762ea 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -395,6 +395,18 @@ public actor SparkConnectClient {
return Self.createPlan { $0.localRelation = localRelation }
}
+ /// Create a `Plan` with a `LocalRelation` containing the given `Apache
Arrow` IPC stream.
+ /// - Parameters:
+ /// - data: An `Apache Arrow` IPC stream.
+ /// - schema: A DDL-formatted schema string.
+ /// - Returns: A `Plan` instance.
+ func getLocalRelation(_ data: Data, _ schema: String) -> Plan {
+ var localRelation = Spark_Connect_LocalRelation()
+ localRelation.data = data
+ localRelation.schema = schema
+ return Self.createPlan { $0.localRelation = localRelation }
+ }
+
/// Create a `Plan` instance for `Range` relation.
/// - Parameters:
/// - start: A start of the range.
diff --git a/Sources/SparkConnect/SparkConnectError.swift
b/Sources/SparkConnect/SparkConnectError.swift
index 45de0a9..5fec560 100644
--- a/Sources/SparkConnect/SparkConnectError.swift
+++ b/Sources/SparkConnect/SparkConnectError.swift
@@ -27,6 +27,7 @@ public enum SparkConnectError: Error {
case InvalidSessionID
case InvalidType
case InvalidViewName
+ case LocalRelationTooLarge
case OutputTypeUnspecified
case ParseSyntaxError
case SchemaNotFound
diff --git a/Sources/SparkConnect/SparkSession.swift
b/Sources/SparkConnect/SparkSession.swift
index 6a366aa..8ae3027 100644
--- a/Sources/SparkConnect/SparkSession.swift
+++ b/Sources/SparkConnect/SparkSession.swift
@@ -108,6 +108,40 @@ public actor SparkSession {
}
}
+ /// The maximum size of a serialized `LocalRelation` which is the default
value of
+ /// `spark.connect.grpc.maxInboundMessageSize` (128MiB).
+ static let maxLocalRelationSize = 128 * 1024 * 1024
+
+ /// Creates a ``DataFrame`` from the given local data with the specified
schema.
+ ///
+ /// ```swift
+ /// let df = try await spark.createDataFrame(
+ /// [[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
+ /// ```
+ ///
+ /// The data is serialized into an `Apache Arrow` IPC stream and embedded
into the plan as a
+ /// `LocalRelation`. The supported schema types are `BOOLEAN`, `TINYINT`,
`SMALLINT`, `INT`,
+ /// `BIGINT`, `FLOAT`, `DOUBLE`, `STRING`, `BINARY`, `DATE`, and
`TIMESTAMP`. `nil` values are
+ /// mapped to `NULL`. Since the serialized data is limited by the Spark
Connect server's gRPC
+ /// message size limit (128MiB by default),
``SparkConnectError/LocalRelationTooLarge`` is
+ /// thrown for larger data.
+ ///
+ /// - Parameters:
+ /// - data: An array of rows whose values are ordered like the schema
fields.
+ /// - schema: A DDL-formatted schema string, e.g. `id INT, name STRING`.
+ /// - Returns: A ``DataFrame`` instance.
+ public func createDataFrame(_ data: [[Sendable?]], _ schema: String) async
throws -> DataFrame {
+ let dataType = try await client.ddlParse(schema)
+ guard case .struct(let structType) = dataType.kind else {
+ throw SparkConnectError.InvalidType
+ }
+ let ipcStream = try ConvertToArrow.toArrowIPCStream(data, structType)
+ guard ipcStream.count <= Self.maxLocalRelationSize else {
+ throw SparkConnectError.LocalRelationTooLarge
+ }
+ return await DataFrame(spark: self, plan:
client.getLocalRelation(ipcStream, schema))
+ }
+
/// Create a ``DataFrame`` with a single `Int64` column name `id`,
containing elements in a
/// range from 0 to `end` (exclusive) with step value 1.
///
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift
b/Tests/SparkConnectTests/CreateDataFrameTests.swift
new file mode 100644
index 0000000..b9a82d9
--- /dev/null
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -0,0 +1,128 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+#if canImport(FoundationEssentials)
+import FoundationEssentials
+#else
+import Foundation
+#endif
+import SparkConnect
+import Testing
+
+/// A test suite for `SparkSession.createDataFrame`
+@Suite(.serialized)
+struct CreateDataFrameTests {
+ @Test
+ func createDataFrame() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.createDataFrame(
+ [[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
+ #expect(try await df.columns == ["id", "name"])
+ #expect(try await df.count() == 3)
+ #expect(try await df.collect() == [Row(1, "Alice"), Row(2, "Bob"), Row(3,
nil)])
+ try await df.show()
+ await spark.stop()
+ }
+
+ @Test
+ func supportedTypes() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let date = Date(timeIntervalSince1970: 86400 * 19_000)
+ let timestamp = Date(timeIntervalSince1970: 1_706_000_000.5)
+ let rows = try await spark.createDataFrame(
+ [
+ [true, Int8(1), Int16(2), 3, Int64(4), Float(1.5), 2.5, "abc", date,
timestamp],
+ [nil, nil, nil, nil, nil, nil, nil, nil, nil, nil],
+ ],
+ "a BOOLEAN, b TINYINT, c SMALLINT, d INT, e BIGINT, f FLOAT, g DOUBLE, h
STRING, i DATE, j TIMESTAMP"
+ ).collect()
+ #expect(
+ rows == [
+ Row(true, 1, 2, 3, 4, Float(1.5), 2.5, "abc", date, timestamp),
+ Row(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func binaryType() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.createDataFrame([[Data([1, 2, 3])], [nil]], "a
BINARY")
+ #expect(try await df.count() == 2)
+ await spark.stop()
+ }
+
+ @Test
+ func emptyData() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.createDataFrame([], "id INT, name STRING")
+ #expect(try await df.columns == ["id", "name"])
+ #expect(try await df.count() == 0)
+ #expect(try await df.collect() == [])
+ await spark.stop()
+ }
+
+ @Test
+ func integerWidening() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.createDataFrame([[Int8(1), 2, Int32(3)]], "a
INT, b BIGINT, c BIGINT")
+ .collect()
+ #expect(rows == [Row(1, 2, 3)])
+ await spark.stop()
+ }
+
+ @Test
+ func invalidTypeValue() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([["a"]], "id INT")
+ }
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([[Int64(Int32.max) + 1]], "id INT")
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func unsupportedType() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([[Decimal(1)]], "id DECIMAL(10, 2)")
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func nonStructSchema() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([[1]], "INT")
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func invalidRowSize() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.InvalidArgument) {
+ try await spark.createDataFrame([[1, 2]], "id INT")
+ }
+ await spark.stop()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]