This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 98035c2 [SPARK-58496] Support field-name-based access in `Row`
98035c2 is described below
commit 98035c23cc34ec7fac3b6f21e85238896dc968c3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Aug 1 15:56:52 2026 -0700
[SPARK-58496] Support field-name-based access in `Row`
### What changes were proposed in this pull request?
This PR aims to support field-name-based access in `Row` like Scala's
`GenericRowWithSchema`.
- A new `RowSchema` class holds the field names and a name-to-index
dictionary, shared by reference across all `Row`s of the same batch.
- `Row` gains `fieldIndex(_:)`, `get(_ name:)`, `subscript(name:)`, and
`asDict()`. For duplicate field names, the last one wins like Scala.
- `DataFrame.collect()` attaches a `RowSchema` created from the Arrow
schema to every `Row`.
### Why are the changes needed?
Previously, `Row` only supported position-based access. This provides the
equivalent of PySpark's `row["name"]`/`asDict()` and Scala's `fieldIndex(name)`.
```swift
let rows = try await spark.sql("SELECT * FROM VALUES (1, 'abc') T(id,
name)").collect()
let id = try rows[0].get("id") as! Int32
let name = try rows[0]["name"] as! String
```
### Does this PR introduce _any_ user-facing change?
Yes, this adds new public APIs. All existing APIs and behaviors are
unchanged.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #467 from dongjoon-hyun/SPARK-58496.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/DataFrame+Actions.swift | 3 +-
Sources/SparkConnect/Row.swift | 62 +++++++++++++++++++++++++++-
Tests/SparkConnectTests/DataFrameTests.swift | 14 +++++++
Tests/SparkConnectTests/RowTests.swift | 56 +++++++++++++++++++++++++
4 files changed, 133 insertions(+), 2 deletions(-)
diff --git a/Sources/SparkConnect/DataFrame+Actions.swift
b/Sources/SparkConnect/DataFrame+Actions.swift
index 455ed38..b57d1ba 100644
--- a/Sources/SparkConnect/DataFrame+Actions.swift
+++ b/Sources/SparkConnect/DataFrame+Actions.swift
@@ -111,6 +111,7 @@ extension DataFrame {
var result: [Row] = []
for batch in self.batches {
+ let rowSchema = RowSchema(batch.schema.fields.map { $0.name })
for i in 0..<batch.length {
var values: [Sendable?] = []
for column in batch.columns {
@@ -163,7 +164,7 @@ extension DataFrame {
}
}
}
- result.append(Row(valueArray: values))
+ result.append(Row(valueArray: values, schema: rowSchema))
}
}
diff --git a/Sources/SparkConnect/Row.swift b/Sources/SparkConnect/Row.swift
index 0481121..caeda7c 100644
--- a/Sources/SparkConnect/Row.swift
+++ b/Sources/SparkConnect/Row.swift
@@ -22,15 +22,34 @@ import FoundationEssentials
import Foundation
#endif
+/// A schema of ``Row`` holding the field names. Like Scala's `StructType`, a
single
+/// `RowSchema` instance is shared by all ``Row``s of the same result.
+public final class RowSchema: Sendable {
+ public let fieldNames: [String]
+ let nameToIndex: [String: Int]
+
+ public init(_ fieldNames: [String]) {
+ self.fieldNames = fieldNames
+ var nameToIndex = [String: Int]()
+ for (index, name) in fieldNames.enumerated() {
+ nameToIndex[name] = index
+ }
+ self.nameToIndex = nameToIndex
+ }
+}
+
public struct Row: Sendable, Equatable {
let values: [Sendable?]
+ let schema: RowSchema?
public init(_ values: Sendable?...) {
self.values = values
+ self.schema = nil
}
- public init(valueArray: [Sendable?]) {
+ public init(valueArray: [Sendable?], schema: RowSchema? = nil) {
self.values = valueArray
+ self.schema = schema
}
public static var empty: Row {
@@ -47,6 +66,12 @@ public struct Row: Sendable, Equatable {
}
}
+ public subscript(name: String) -> Sendable {
+ get throws {
+ return try get(name)
+ }
+ }
+
public func get(_ i: Int) throws -> Sendable {
if i < 0 || i >= self.length {
throw SparkConnectError.InvalidArgument
@@ -54,6 +79,41 @@ public struct Row: Sendable, Equatable {
return values[i]
}
+ /// Returns the value of the field with the given name.
+ /// - Parameter name: A field name.
+ /// - Returns: A value of the field.
+ public func get(_ name: String) throws -> Sendable {
+ return values[try fieldIndex(name)]
+ }
+
+ /// Returns the index of the field with the given name. If multiple fields
have the same
+ /// name, the last one is returned like Scala's `StructType.fieldIndex`.
+ /// - Parameter name: A field name.
+ /// - Returns: An index of the field.
+ public func fieldIndex(_ name: String) throws -> Int {
+ guard let schema = self.schema else {
+ throw SparkConnectError.UnsupportedOperation
+ }
+ guard let index = schema.nameToIndex[name] else {
+ throw SparkConnectError.ColumnNotFound
+ }
+ return index
+ }
+
+ /// Returns the row as a dictionary from field names to values. If multiple
fields have
+ /// the same name, the last value is used.
+ /// - Returns: A dictionary from field names to values.
+ public func asDict() throws -> [String: Sendable?] {
+ guard let schema = self.schema else {
+ throw SparkConnectError.UnsupportedOperation
+ }
+ var dict = [String: Sendable?]()
+ for (index, name) in schema.fieldNames.enumerated() {
+ dict.updateValue(values[index], forKey: name)
+ }
+ return dict
+ }
+
public func getAsBool(_ i: Int) throws -> Bool {
return try get(i) as! Bool
}
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index c49b7dc..a23b4b4 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -533,6 +533,20 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func collectByName() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.sql(
+ "SELECT * FROM VALUES (1, 'abc'), (2, 'def') T(id, name)"
+ ).collect()
+ #expect(try rows[0].get("id") as! Int32 == 1)
+ #expect(try rows[0].get("name") as! String == "abc")
+ #expect(try rows[1]["id"] as! Int32 == 2)
+ #expect(try rows[1]["name"] as! String == "def")
+ #expect(try rows[0].fieldIndex("name") == 1)
+ await spark.stop()
+ }
+
@Test
func collectMultiple() async throws {
let spark = try await SparkSession.builder.getOrCreate()
diff --git a/Tests/SparkConnectTests/RowTests.swift
b/Tests/SparkConnectTests/RowTests.swift
index 965e230..6932e06 100644
--- a/Tests/SparkConnectTests/RowTests.swift
+++ b/Tests/SparkConnectTests/RowTests.swift
@@ -66,6 +66,62 @@ struct RowTests {
}
}
+ @Test
+ func fieldIndex() throws {
+ let row = Row(valueArray: [1, "a"], schema: RowSchema(["id", "name"]))
+ #expect(try row.fieldIndex("id") == 0)
+ #expect(try row.fieldIndex("name") == 1)
+ #expect(throws: SparkConnectError.ColumnNotFound) {
+ try row.fieldIndex("nonexistent")
+ }
+ #expect(throws: SparkConnectError.UnsupportedOperation) {
+ try Row(1).fieldIndex("id")
+ }
+ }
+
+ @Test
+ func getByName() throws {
+ let row = Row(valueArray: [1, 1.1, "a", true], schema: RowSchema(["id",
"value", "name", "flag"]))
+ #expect(try row.get("id") as! Int == 1)
+ #expect(try row.get("value") as! Double == 1.1)
+ #expect(try row.get("name") as! String == "a")
+ #expect(try row.get("flag") as! Bool == true)
+ #expect(try row["id"] as! Int == 1)
+ #expect(try row["name"] as! String == "a")
+ #expect(throws: SparkConnectError.ColumnNotFound) {
+ try row.get("nonexistent")
+ }
+ }
+
+ @Test
+ func duplicateFieldNames() throws {
+ // Like Scala's `StructType.fieldIndex`, the last field wins for duplicate
names.
+ let row = Row(valueArray: [1, 2], schema: RowSchema(["id", "id"]))
+ #expect(try row.fieldIndex("id") == 1)
+ #expect(try row.get("id") as! Int == 2)
+ }
+
+ @Test
+ func asDict() throws {
+ let row = Row(valueArray: [1, "a", nil], schema: RowSchema(["id", "name",
"none"]))
+ let dict = try row.asDict()
+ #expect(dict.count == 3)
+ #expect(dict["id"] as! Int == 1)
+ #expect(dict["name"] as! String == "a")
+ #expect(dict.keys.contains("none"))
+ #expect(throws: SparkConnectError.UnsupportedOperation) {
+ try Row(1).asDict()
+ }
+ }
+
+ @Test
+ func compareIgnoresSchema() {
+ #expect(Row(valueArray: [1, "a"], schema: RowSchema(["id", "name"])) ==
Row(1, "a"))
+ #expect(
+ Row(valueArray: [1], schema: RowSchema(["a"]))
+ == Row(valueArray: [1], schema: RowSchema(["b"])))
+ }
+
@Test
func compare() {
#expect(Row(nil) != Row())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]