This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new ab62a17  [SPARK-58497] Support `createDataFrame` in `SparkSession`
ab62a17 is described below

commit ab62a1766e540d283cc4d634c44a071e752025a5
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Aug 1 15:57:51 2026 -0700

    [SPARK-58497] Support `createDataFrame` in `SparkSession`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `SparkSession.createDataFrame(_:_:)` to create a 
`DataFrame` from local Swift data with a DDL-formatted schema.
    
    ```swift
    let df = try await spark.createDataFrame(
      [[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
    ```
    
    The data is serialized into an `Apache Arrow` IPC stream via the in-tree 
`ArrowWriter` and embedded into the plan as a `LocalRelation`. Supported types 
are `BOOLEAN`, `TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, 
`STRING`, `BINARY`, `DATE`, and `TIMESTAMP`. `nil` is mapped to `NULL`. Data 
larger than 128MiB throws the new `SparkConnectError.LocalRelationTooLarge`.
    
    ### Why are the changes needed?
    
    Previously, there was no way to create a `DataFrame` from local Swift data. 
This is one of the fundamental `SparkSession` APIs which all other Spark 
Connect clients provide.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a new API addition.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added `CreateDataFrameTests` test suite.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #468 from dongjoon-hyun/SPARK-58497.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/ConvertToArrow.swift          | 129 +++++++++++++++++++++
 Sources/SparkConnect/SparkConnectClient.swift      |  12 ++
 Sources/SparkConnect/SparkConnectError.swift       |   1 +
 Sources/SparkConnect/SparkSession.swift            |  34 ++++++
 Tests/SparkConnectTests/CreateDataFrameTests.swift | 128 ++++++++++++++++++++
 5 files changed, 304 insertions(+)

diff --git a/Sources/SparkConnect/ConvertToArrow.swift 
b/Sources/SparkConnect/ConvertToArrow.swift
new file mode 100644
index 0000000..6473282
--- /dev/null
+++ b/Sources/SparkConnect/ConvertToArrow.swift
@@ -0,0 +1,129 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+#if canImport(FoundationEssentials)
+import FoundationEssentials
+#else
+import Foundation
+#endif
+
+/// A utility to convert local data into an `Apache Arrow` IPC stream for 
`LocalRelation`.
+enum ConvertToArrow {
+  /// Convert the given rows into an `Apache Arrow` IPC stream according to 
the Spark schema.
+  /// - Parameters:
+  ///   - data: An array of rows whose values are ordered like the schema 
fields.
+  ///   - schema: A ``StructType`` describing the field names and types.
+  /// - Returns: A serialized `Apache Arrow` IPC stream.
+  static func toArrowIPCStream(_ data: [[Sendable?]], _ schema: StructType) 
throws -> Data {
+    for row in data where row.count != schema.fields.count {
+      throw SparkConnectError.InvalidArgument
+    }
+    let batchBuilder = RecordBatch.Builder()
+    for (i, field) in schema.fields.enumerated() {
+      let column = data.map { $0[i] }
+      let holder = try toArrowColumn(column, field.dataType)
+      batchBuilder.addColumn(
+        ArrowField(field.name, type: holder.type, isNullable: field.nullable), 
arrowArray: holder)
+    }
+    switch batchBuilder.finish() {
+    case .success(let batch):
+      switch ArrowWriter().writeStreaming(
+        ArrowWriter.Info(.recordbatch, schema: batch.schema, batches: [batch]))
+      {
+      case .success(let ipcStream):
+        return ipcStream
+      case .failure:
+        throw SparkConnectError.InvalidArrowData
+      }
+    case .failure:
+      throw SparkConnectError.InvalidArrowData
+    }
+  }
+
+  /// Build an Arrow column from the given values according to the Spark data 
type.
+  private static func toArrowColumn(_ column: [Sendable?], _ dataType: 
DataType) throws
+    -> ArrowArrayHolder
+  {
+    switch dataType.kind {
+    case .boolean:
+      return try fill(ArrowArrayBuilders.loadBoolArrayBuilder(), column) { $0 
as? Bool }
+    case .byte:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Int8>,
+        column) { toInt64($0).flatMap { Int8(exactly: $0) } }
+    case .short:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Int16>,
+        column) { toInt64($0).flatMap { Int16(exactly: $0) } }
+    case .integer:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Int32>,
+        column) { toInt64($0).flatMap { Int32(exactly: $0) } }
+    case .long:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Int64>,
+        column) { toInt64($0) }
+    case .float:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Float>,
+        column) { $0 as? Float }
+    case .double:
+      return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as 
NumberArrayBuilder<Double>,
+        column) { ($0 as? Double) ?? ($0 as? Float).map(Double.init) }
+    case .string:
+      return try fill(ArrowArrayBuilders.loadStringArrayBuilder(), column) { 
$0 as? String }
+    case .binary:
+      return try fill(ArrowArrayBuilders.loadBinaryArrayBuilder(), column) { 
$0 as? Data }
+    case .date:
+      return try fill(ArrowArrayBuilders.loadDate32ArrayBuilder(), column) { 
$0 as? Date }
+    case .timestamp:
+      return try fill(
+        ArrowArrayBuilders.loadTimestampArrayBuilder(.microseconds, timezone: 
"UTC"), column
+      ) { ($0 as? Date).map { Int64(($0.timeIntervalSince1970 * 
1_000_000).rounded()) } }
+    default:
+      throw SparkConnectError.InvalidType
+    }
+  }
+
+  /// Append the given values to the builder while mapping `nil` to `null`.
+  /// Unlike `appendAny`, a value which is not convertible throws 
``SparkConnectError/InvalidType``
+  /// instead of being appended as `null` silently.
+  private static func fill<T, U>(
+    _ builder: ArrowArrayBuilder<T, U>, _ column: [Sendable?],
+    _ convert: (Sendable) -> T.ItemType?
+  ) throws -> ArrowArrayHolder {
+    for value in column {
+      if let value {
+        guard let converted = convert(value) else {
+          throw SparkConnectError.InvalidType
+        }
+        builder.append(converted)
+      } else {
+        builder.append(nil)
+      }
+    }
+    return try builder.toHolder()
+  }
+
+  private static func toInt64(_ value: Sendable) -> Int64? {
+    switch value {
+    case let v as Int: return Int64(v)
+    case let v as Int8: return Int64(v)
+    case let v as Int16: return Int64(v)
+    case let v as Int32: return Int64(v)
+    case let v as Int64: return v
+    default: return nil
+    }
+  }
+}
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index e545766..aa762ea 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -395,6 +395,18 @@ public actor SparkConnectClient {
     return Self.createPlan { $0.localRelation = localRelation }
   }
 
+  /// Create a `Plan` with a `LocalRelation` containing the given `Apache 
Arrow` IPC stream.
+  /// - Parameters:
+  ///   - data: An `Apache Arrow` IPC stream.
+  ///   - schema: A DDL-formatted schema string.
+  /// - Returns: A `Plan` instance.
+  func getLocalRelation(_ data: Data, _ schema: String) -> Plan {
+    var localRelation = Spark_Connect_LocalRelation()
+    localRelation.data = data
+    localRelation.schema = schema
+    return Self.createPlan { $0.localRelation = localRelation }
+  }
+
   /// Create a `Plan` instance for `Range` relation.
   /// - Parameters:
   ///   - start: A start of the range.
diff --git a/Sources/SparkConnect/SparkConnectError.swift 
b/Sources/SparkConnect/SparkConnectError.swift
index 45de0a9..5fec560 100644
--- a/Sources/SparkConnect/SparkConnectError.swift
+++ b/Sources/SparkConnect/SparkConnectError.swift
@@ -27,6 +27,7 @@ public enum SparkConnectError: Error {
   case InvalidSessionID
   case InvalidType
   case InvalidViewName
+  case LocalRelationTooLarge
   case OutputTypeUnspecified
   case ParseSyntaxError
   case SchemaNotFound
diff --git a/Sources/SparkConnect/SparkSession.swift 
b/Sources/SparkConnect/SparkSession.swift
index 6a366aa..8ae3027 100644
--- a/Sources/SparkConnect/SparkSession.swift
+++ b/Sources/SparkConnect/SparkSession.swift
@@ -108,6 +108,40 @@ public actor SparkSession {
     }
   }
 
+  /// The maximum size of a serialized `LocalRelation` which is the default 
value of
+  /// `spark.connect.grpc.maxInboundMessageSize` (128MiB).
+  static let maxLocalRelationSize = 128 * 1024 * 1024
+
+  /// Creates a ``DataFrame`` from the given local data with the specified 
schema.
+  ///
+  /// ```swift
+  /// let df = try await spark.createDataFrame(
+  ///   [[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
+  /// ```
+  ///
+  /// The data is serialized into an `Apache Arrow` IPC stream and embedded 
into the plan as a
+  /// `LocalRelation`. The supported schema types are `BOOLEAN`, `TINYINT`, 
`SMALLINT`, `INT`,
+  /// `BIGINT`, `FLOAT`, `DOUBLE`, `STRING`, `BINARY`, `DATE`, and 
`TIMESTAMP`. `nil` values are
+  /// mapped to `NULL`. Since the serialized data is limited by the Spark 
Connect server's gRPC
+  /// message size limit (128MiB by default), 
``SparkConnectError/LocalRelationTooLarge`` is
+  /// thrown for larger data.
+  ///
+  /// - Parameters:
+  ///   - data: An array of rows whose values are ordered like the schema 
fields.
+  ///   - schema: A DDL-formatted schema string, e.g. `id INT, name STRING`.
+  /// - Returns: A ``DataFrame`` instance.
+  public func createDataFrame(_ data: [[Sendable?]], _ schema: String) async 
throws -> DataFrame {
+    let dataType = try await client.ddlParse(schema)
+    guard case .struct(let structType) = dataType.kind else {
+      throw SparkConnectError.InvalidType
+    }
+    let ipcStream = try ConvertToArrow.toArrowIPCStream(data, structType)
+    guard ipcStream.count <= Self.maxLocalRelationSize else {
+      throw SparkConnectError.LocalRelationTooLarge
+    }
+    return await DataFrame(spark: self, plan: 
client.getLocalRelation(ipcStream, schema))
+  }
+
   /// Create a ``DataFrame`` with a single `Int64` column name `id`, 
containing elements in a
   /// range from 0 to `end` (exclusive) with step value 1.
   ///
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift 
b/Tests/SparkConnectTests/CreateDataFrameTests.swift
new file mode 100644
index 0000000..b9a82d9
--- /dev/null
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -0,0 +1,128 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+#if canImport(FoundationEssentials)
+import FoundationEssentials
+#else
+import Foundation
+#endif
+import SparkConnect
+import Testing
+
+/// A test suite for `SparkSession.createDataFrame`
+@Suite(.serialized)
+struct CreateDataFrameTests {
+  @Test
+  func createDataFrame() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.createDataFrame(
+      [[1, "Alice"], [2, "Bob"], [3, nil]], "id INT, name STRING")
+    #expect(try await df.columns == ["id", "name"])
+    #expect(try await df.count() == 3)
+    #expect(try await df.collect() == [Row(1, "Alice"), Row(2, "Bob"), Row(3, 
nil)])
+    try await df.show()
+    await spark.stop()
+  }
+
+  @Test
+  func supportedTypes() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let date = Date(timeIntervalSince1970: 86400 * 19_000)
+    let timestamp = Date(timeIntervalSince1970: 1_706_000_000.5)
+    let rows = try await spark.createDataFrame(
+      [
+        [true, Int8(1), Int16(2), 3, Int64(4), Float(1.5), 2.5, "abc", date, 
timestamp],
+        [nil, nil, nil, nil, nil, nil, nil, nil, nil, nil],
+      ],
+      "a BOOLEAN, b TINYINT, c SMALLINT, d INT, e BIGINT, f FLOAT, g DOUBLE, h 
STRING, i DATE, j TIMESTAMP"
+    ).collect()
+    #expect(
+      rows == [
+        Row(true, 1, 2, 3, 4, Float(1.5), 2.5, "abc", date, timestamp),
+        Row(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil),
+      ])
+    await spark.stop()
+  }
+
+  @Test
+  func binaryType() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.createDataFrame([[Data([1, 2, 3])], [nil]], "a 
BINARY")
+    #expect(try await df.count() == 2)
+    await spark.stop()
+  }
+
+  @Test
+  func emptyData() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.createDataFrame([], "id INT, name STRING")
+    #expect(try await df.columns == ["id", "name"])
+    #expect(try await df.count() == 0)
+    #expect(try await df.collect() == [])
+    await spark.stop()
+  }
+
+  @Test
+  func integerWidening() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await spark.createDataFrame([[Int8(1), 2, Int32(3)]], "a 
INT, b BIGINT, c BIGINT")
+      .collect()
+    #expect(rows == [Row(1, 2, 3)])
+    await spark.stop()
+  }
+
+  @Test
+  func invalidTypeValue() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    await #expect(throws: SparkConnectError.InvalidType) {
+      try await spark.createDataFrame([["a"]], "id INT")
+    }
+    await #expect(throws: SparkConnectError.InvalidType) {
+      try await spark.createDataFrame([[Int64(Int32.max) + 1]], "id INT")
+    }
+    await spark.stop()
+  }
+
+  @Test
+  func unsupportedType() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    await #expect(throws: SparkConnectError.InvalidType) {
+      try await spark.createDataFrame([[Decimal(1)]], "id DECIMAL(10, 2)")
+    }
+    await spark.stop()
+  }
+
+  @Test
+  func nonStructSchema() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    await #expect(throws: SparkConnectError.InvalidType) {
+      try await spark.createDataFrame([[1]], "INT")
+    }
+    await spark.stop()
+  }
+
+  @Test
+  func invalidRowSize() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    await #expect(throws: SparkConnectError.InvalidArgument) {
+      try await spark.createDataFrame([[1, 2]], "id INT")
+    }
+    await spark.stop()
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to