This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 98035c2  [SPARK-58496] Support field-name-based access in `Row`
98035c2 is described below

commit 98035c23cc34ec7fac3b6f21e85238896dc968c3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Aug 1 15:56:52 2026 -0700

    [SPARK-58496] Support field-name-based access in `Row`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support field-name-based access in `Row` like Scala's 
`GenericRowWithSchema`.
    
    - A new `RowSchema` class holds the field names and a name-to-index 
dictionary, shared by reference across all `Row`s of the same batch.
    - `Row` gains `fieldIndex(_:)`, `get(_ name:)`, `subscript(name:)`, and 
`asDict()`. For duplicate field names, the last one wins like Scala.
    - `DataFrame.collect()` attaches a `RowSchema` created from the Arrow 
schema to every `Row`.
    
    ### Why are the changes needed?
    
    Previously, `Row` only supported position-based access. This provides the 
equivalent of PySpark's `row["name"]`/`asDict()` and Scala's `fieldIndex(name)`.
    
    ```swift
    let rows = try await spark.sql("SELECT * FROM VALUES (1, 'abc') T(id, 
name)").collect()
    let id = try rows[0].get("id") as! Int32
    let name = try rows[0]["name"] as! String
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, this adds new public APIs. All existing APIs and behaviors are 
unchanged.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #467 from dongjoon-hyun/SPARK-58496.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/DataFrame+Actions.swift |  3 +-
 Sources/SparkConnect/Row.swift               | 62 +++++++++++++++++++++++++++-
 Tests/SparkConnectTests/DataFrameTests.swift | 14 +++++++
 Tests/SparkConnectTests/RowTests.swift       | 56 +++++++++++++++++++++++++
 4 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/Sources/SparkConnect/DataFrame+Actions.swift 
b/Sources/SparkConnect/DataFrame+Actions.swift
index 455ed38..b57d1ba 100644
--- a/Sources/SparkConnect/DataFrame+Actions.swift
+++ b/Sources/SparkConnect/DataFrame+Actions.swift
@@ -111,6 +111,7 @@ extension DataFrame {
 
     var result: [Row] = []
     for batch in self.batches {
+      let rowSchema = RowSchema(batch.schema.fields.map { $0.name })
       for i in 0..<batch.length {
         var values: [Sendable?] = []
         for column in batch.columns {
@@ -163,7 +164,7 @@ extension DataFrame {
             }
           }
         }
-        result.append(Row(valueArray: values))
+        result.append(Row(valueArray: values, schema: rowSchema))
       }
     }
 
diff --git a/Sources/SparkConnect/Row.swift b/Sources/SparkConnect/Row.swift
index 0481121..caeda7c 100644
--- a/Sources/SparkConnect/Row.swift
+++ b/Sources/SparkConnect/Row.swift
@@ -22,15 +22,34 @@ import FoundationEssentials
 import Foundation
 #endif
 
+/// A schema of ``Row`` holding the field names. Like Scala's `StructType`, a 
single
+/// `RowSchema` instance is shared by all ``Row``s of the same result.
+public final class RowSchema: Sendable {
+  public let fieldNames: [String]
+  let nameToIndex: [String: Int]
+
+  public init(_ fieldNames: [String]) {
+    self.fieldNames = fieldNames
+    var nameToIndex = [String: Int]()
+    for (index, name) in fieldNames.enumerated() {
+      nameToIndex[name] = index
+    }
+    self.nameToIndex = nameToIndex
+  }
+}
+
 public struct Row: Sendable, Equatable {
   let values: [Sendable?]
+  let schema: RowSchema?
 
   public init(_ values: Sendable?...) {
     self.values = values
+    self.schema = nil
   }
 
-  public init(valueArray: [Sendable?]) {
+  public init(valueArray: [Sendable?], schema: RowSchema? = nil) {
     self.values = valueArray
+    self.schema = schema
   }
 
   public static var empty: Row {
@@ -47,6 +66,12 @@ public struct Row: Sendable, Equatable {
     }
   }
 
+  public subscript(name: String) -> Sendable {
+    get throws {
+      return try get(name)
+    }
+  }
+
   public func get(_ i: Int) throws -> Sendable {
     if i < 0 || i >= self.length {
       throw SparkConnectError.InvalidArgument
@@ -54,6 +79,41 @@ public struct Row: Sendable, Equatable {
     return values[i]
   }
 
+  /// Returns the value of the field with the given name.
+  /// - Parameter name: A field name.
+  /// - Returns: A value of the field.
+  public func get(_ name: String) throws -> Sendable {
+    return values[try fieldIndex(name)]
+  }
+
+  /// Returns the index of the field with the given name. If multiple fields 
have the same
+  /// name, the last one is returned like Scala's `StructType.fieldIndex`.
+  /// - Parameter name: A field name.
+  /// - Returns: An index of the field.
+  public func fieldIndex(_ name: String) throws -> Int {
+    guard let schema = self.schema else {
+      throw SparkConnectError.UnsupportedOperation
+    }
+    guard let index = schema.nameToIndex[name] else {
+      throw SparkConnectError.ColumnNotFound
+    }
+    return index
+  }
+
+  /// Returns the row as a dictionary from field names to values. If multiple 
fields have
+  /// the same name, the last value is used.
+  /// - Returns: A dictionary from field names to values.
+  public func asDict() throws -> [String: Sendable?] {
+    guard let schema = self.schema else {
+      throw SparkConnectError.UnsupportedOperation
+    }
+    var dict = [String: Sendable?]()
+    for (index, name) in schema.fieldNames.enumerated() {
+      dict.updateValue(values[index], forKey: name)
+    }
+    return dict
+  }
+
   public func getAsBool(_ i: Int) throws -> Bool {
     return try get(i) as! Bool
   }
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index c49b7dc..a23b4b4 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -533,6 +533,20 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func collectByName() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await spark.sql(
+      "SELECT * FROM VALUES (1, 'abc'), (2, 'def') T(id, name)"
+    ).collect()
+    #expect(try rows[0].get("id") as! Int32 == 1)
+    #expect(try rows[0].get("name") as! String == "abc")
+    #expect(try rows[1]["id"] as! Int32 == 2)
+    #expect(try rows[1]["name"] as! String == "def")
+    #expect(try rows[0].fieldIndex("name") == 1)
+    await spark.stop()
+  }
+
   @Test
   func collectMultiple() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
diff --git a/Tests/SparkConnectTests/RowTests.swift 
b/Tests/SparkConnectTests/RowTests.swift
index 965e230..6932e06 100644
--- a/Tests/SparkConnectTests/RowTests.swift
+++ b/Tests/SparkConnectTests/RowTests.swift
@@ -66,6 +66,62 @@ struct RowTests {
     }
   }
 
+  @Test
+  func fieldIndex() throws {
+    let row = Row(valueArray: [1, "a"], schema: RowSchema(["id", "name"]))
+    #expect(try row.fieldIndex("id") == 0)
+    #expect(try row.fieldIndex("name") == 1)
+    #expect(throws: SparkConnectError.ColumnNotFound) {
+      try row.fieldIndex("nonexistent")
+    }
+    #expect(throws: SparkConnectError.UnsupportedOperation) {
+      try Row(1).fieldIndex("id")
+    }
+  }
+
+  @Test
+  func getByName() throws {
+    let row = Row(valueArray: [1, 1.1, "a", true], schema: RowSchema(["id", 
"value", "name", "flag"]))
+    #expect(try row.get("id") as! Int == 1)
+    #expect(try row.get("value") as! Double == 1.1)
+    #expect(try row.get("name") as! String == "a")
+    #expect(try row.get("flag") as! Bool == true)
+    #expect(try row["id"] as! Int == 1)
+    #expect(try row["name"] as! String == "a")
+    #expect(throws: SparkConnectError.ColumnNotFound) {
+      try row.get("nonexistent")
+    }
+  }
+
+  @Test
+  func duplicateFieldNames() throws {
+    // Like Scala's `StructType.fieldIndex`, the last field wins for duplicate 
names.
+    let row = Row(valueArray: [1, 2], schema: RowSchema(["id", "id"]))
+    #expect(try row.fieldIndex("id") == 1)
+    #expect(try row.get("id") as! Int == 2)
+  }
+
+  @Test
+  func asDict() throws {
+    let row = Row(valueArray: [1, "a", nil], schema: RowSchema(["id", "name", 
"none"]))
+    let dict = try row.asDict()
+    #expect(dict.count == 3)
+    #expect(dict["id"] as! Int == 1)
+    #expect(dict["name"] as! String == "a")
+    #expect(dict.keys.contains("none"))
+    #expect(throws: SparkConnectError.UnsupportedOperation) {
+      try Row(1).asDict()
+    }
+  }
+
+  @Test
+  func compareIgnoresSchema() {
+    #expect(Row(valueArray: [1, "a"], schema: RowSchema(["id", "name"])) == 
Row(1, "a"))
+    #expect(
+      Row(valueArray: [1], schema: RowSchema(["a"]))
+        == Row(valueArray: [1], schema: RowSchema(["b"])))
+  }
+
   @Test
   func compare() {
     #expect(Row(nil) != Row())


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to