This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new cceb2e4  [SPARK-58476] Support Column-based 
`sort/withColumn/drop/join` in `DataFrame`
cceb2e4 is described below

commit cceb2e47b37e6219d094ddcea6436dc216707177
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 10:36:55 2026 -0700

    [SPARK-58476] Support Column-based `sort/withColumn/drop/join` in 
`DataFrame`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `Column`-based overloads for `sort`, `orderBy`, 
`sortWithinPartitions`, `withColumn`, `drop`, and `join` in `DataFrame`.
    
    Previously, only `select`/`agg`/`filter`/`where`/`observe` accepted 
`Column` expressions. This PR extends the remaining common entry points to 
accept `Column`, matching the Scala client `Dataset` signatures:
    
    - `sort(_ col: Column, _ cols: Column...)`, `orderBy(_ col: Column, _ cols: 
Column...)`, `sortWithinPartitions(_ col: Column, _ cols: Column...)` — a 
`SortOrder` expression (e.g., `col("id").desc()`) is used as-is; a plain column 
is wrapped in ascending order with nulls first, like the Scala client's 
`Column.sortOrder`.
    - `withColumn(_ colName: String, _ col: Column)` — builds an `Alias` 
expression for the `WithColumns` relation.
    - `drop(_ col: Column, _ cols: Column...)` — uses the `columns` 
(expression) field of the `Drop` relation.
    - `join(_ right: DataFrame, joinExprs: Column)` and `join(_ right: 
DataFrame, joinExprs: Column, joinType: String)` — passes the column expression 
as the `join_condition`.
    
    All existing `String`-based overloads are kept intact.
    
    ### Why are the changes needed?
    
    To improve the usability of the `Column` API (`col`, `lit`, operators, 
`asc`/`desc`) by allowing it in the main `DataFrame` transformation entry 
points, for feature parity with the Scala/PySpark clients.
    
    ```swift
    df.sort(col("department"), col("salary").desc())
    df.withColumn("doubled", col("id") * 2)
    df.drop(col("a"), col("b"))
    df1.alias("l").join(df2.alias("r"), joinExprs: col("l.id") == col("r.id"), 
joinType: "left")
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, this adds new APIs. The existing string-based APIs are unchanged.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #458 from dongjoon-hyun/SPARK-58476.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../SparkConnect/DataFrame+Transformations.swift   | 108 +++++++++++++++++++++
 Sources/SparkConnect/SparkConnectClient.swift      |  50 ++++++++++
 Tests/SparkConnectTests/DataFrameTests.swift       |  82 ++++++++++++++++
 3 files changed, 240 insertions(+)

diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift 
b/Sources/SparkConnect/DataFrame+Transformations.swift
index 674d223..0134181 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -109,6 +109,21 @@ extension DataFrame {
     return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getDrop(self.plan.root, cols))
   }
 
+  /// Returns a new Dataset with columns dropped.
+  /// This is a no-op if schema doesn't contain the column expressions.
+  ///
+  /// ```swift
+  /// let df2 = df.drop(col("a"), col("b"))
+  /// ```
+  /// - Parameters:
+  ///   - col: A ``Column`` to drop.
+  ///   - cols: Additional ``Column``s to drop.
+  /// - Returns: A ``DataFrame`` without the given columns.
+  public func drop(_ col: Column, _ cols: Column...) -> DataFrame {
+    let exprs = ([col] + cols).map { $0.expr }
+    return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getDrop(self.plan.root, exprs))
+  }
+
   /// Returns a new ``DataFrame`` that contains only the unique rows from this 
``DataFrame``.
   /// This is an alias for `distinct`. If column names are given, Spark 
considers only those columns.
   /// - Parameter cols: Column names
@@ -185,6 +200,22 @@ extension DataFrame {
     return withColumns([colName: expr])
   }
 
+  /// Returns a new ``DataFrame`` by adding a column or replacing the existing 
column that has the
+  /// same name.
+  ///
+  /// ```swift
+  /// let df2 = df.withColumn("doubled", col("id") * 2)
+  /// ```
+  /// - Parameters:
+  ///   - colName: A new column name.
+  ///   - col: A ``Column`` expression for the new column.
+  /// - Returns: A ``DataFrame`` with the new or replaced column.
+  public func withColumn(_ colName: String, _ col: Column) -> DataFrame {
+    return DataFrame(
+      spark: self.spark,
+      plan: SparkConnectClient.getWithColumns(self.plan.root, colName, 
col.expr))
+  }
+
   /// Returns a new ``DataFrame`` by adding columns or replacing the existing 
columns that have the
   /// same names.
   /// - Parameter colsMap: A dictionary of column name and SQL expression 
string.
@@ -280,6 +311,21 @@ extension DataFrame {
     return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getSort(self.plan.root, cols))
   }
 
+  /// Returns a new ``DataFrame`` sorted by the specified ``Column`` 
expressions.
+  /// Columns without an explicit sort order are sorted in ascending order.
+  ///
+  /// ```swift
+  /// let sorted = df.sort(col("department"), col("salary").desc())
+  /// ```
+  /// - Parameters:
+  ///   - col: A ``Column`` or sort order expression to sort by.
+  ///   - cols: Additional ``Column`` or sort order expressions.
+  /// - Returns: A new DataFrame sorted by the specified columns
+  public func sort(_ col: Column, _ cols: Column...) -> DataFrame {
+    let exprs = ([col] + cols).map { $0.expr }
+    return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getSort(self.plan.root, exprs))
+  }
+
   /// Return a new ``DataFrame`` sorted by the specified column(s).
   /// - Parameter cols: Column names.
   /// - Returns: A sorted ``DataFrame``
@@ -287,6 +333,21 @@ extension DataFrame {
     return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getSort(self.plan.root, cols))
   }
 
+  /// Return a new ``DataFrame`` sorted by the specified ``Column`` 
expressions.
+  /// Columns without an explicit sort order are sorted in ascending order.
+  ///
+  /// ```swift
+  /// let sorted = df.orderBy(col("id").desc())
+  /// ```
+  /// - Parameters:
+  ///   - col: A ``Column`` or sort order expression to sort by.
+  ///   - cols: Additional ``Column`` or sort order expressions.
+  /// - Returns: A sorted ``DataFrame``
+  public func orderBy(_ col: Column, _ cols: Column...) -> DataFrame {
+    let exprs = ([col] + cols).map { $0.expr }
+    return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getSort(self.plan.root, exprs))
+  }
+
   /// Returns a new ``DataFrame`` with each partition sorted by the specified 
column(s).
   /// This is the same operation as `SORT BY` in SQL (Hive QL).
   /// - Parameter cols: Column names.
@@ -296,6 +357,23 @@ extension DataFrame {
       spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root, 
cols, false))
   }
 
+  /// Returns a new ``DataFrame`` with each partition sorted by the specified 
``Column``
+  /// expressions. Columns without an explicit sort order are sorted in 
ascending order.
+  /// This is the same operation as `SORT BY` in SQL (Hive QL).
+  ///
+  /// ```swift
+  /// let sorted = df.sortWithinPartitions(col("id").desc())
+  /// ```
+  /// - Parameters:
+  ///   - col: A ``Column`` or sort order expression to sort by.
+  ///   - cols: Additional ``Column`` or sort order expressions.
+  /// - Returns: A ``DataFrame`` with each partition sorted.
+  public func sortWithinPartitions(_ col: Column, _ cols: Column...) -> 
DataFrame {
+    let exprs = ([col] + cols).map { $0.expr }
+    return DataFrame(
+      spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root, 
exprs, false))
+  }
+
   /// Limits the result count to the number specified.
   ///
   /// This transformation is often used for:
@@ -505,6 +583,36 @@ extension DataFrame {
     return DataFrame(spark: self.spark, plan: plan)
   }
 
+  /// Inner join with another `DataFrame`, using the given join expression.
+  ///
+  /// ```swift
+  /// let joined = df1.alias("a").join(df2.alias("b"), joinExprs: col("a.id") 
== col("b.id"))
+  /// ```
+  /// - Parameters:
+  ///   - right: Right side of the join operation.
+  ///   - joinExprs: A join expression ``Column``.
+  /// - Returns: A `DataFrame`.
+  public func join(_ right: DataFrame, joinExprs: Column) async -> DataFrame {
+    return await join(right, joinExprs: joinExprs, joinType: "inner")
+  }
+
+  /// Join with another `DataFrame`, using the given join expression and join 
type.
+  /// - Parameters:
+  ///   - right: Right side of the join operation.
+  ///   - joinExprs: A join expression ``Column``.
+  ///   - joinType: A join type name.
+  /// - Returns: A `DataFrame`.
+  public func join(_ right: DataFrame, joinExprs: Column, joinType: String) 
async -> DataFrame {
+    let rightPlan = await (right.getPlan() as! Plan).root
+    let plan = SparkConnectClient.getJoin(
+      self.plan.root,
+      rightPlan,
+      joinType.toJoinType,
+      joinCondition: joinExprs.expr
+    )
+    return DataFrame(spark: self.spark, plan: plan)
+  }
+
   /// Explicit cartesian join with another `DataFrame`.
   /// - Parameter right: Right side of the join operation.
   /// - Returns: A `DataFrame`.
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index ff00e27..e545766 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -595,6 +595,18 @@ public actor SparkConnectClient {
     return createPlan { $0.withColumns = withColumns }
   }
 
+  static func getWithColumns(
+    _ child: Relation, _ colName: String, _ expr: Spark_Connect_Expression
+  ) -> Plan {
+    var withColumns = WithColumns()
+    withColumns.input = child
+    var alias = Spark_Connect_Expression.Alias()
+    alias.expr = expr
+    alias.name = [colName]
+    withColumns.aliases = [alias]
+    return createPlan { $0.withColumns = withColumns }
+  }
+
   static func getFilter(_ child: Relation, _ conditionExpr: String) -> Plan {
     var filter = Filter()
     filter.input = child
@@ -616,6 +628,13 @@ public actor SparkConnectClient {
     return createPlan { $0.drop = drop }
   }
 
+  static func getDrop(_ child: Relation, _ columns: 
[Spark_Connect_Expression]) -> Plan {
+    var drop = Drop()
+    drop.input = child
+    drop.columns = columns
+    return createPlan { $0.drop = drop }
+  }
+
   static func getDropDuplicates(
     _ child: Relation,
     _ columnNames: [String],
@@ -723,6 +742,25 @@ public actor SparkConnectClient {
     return createPlan { $0.sort = sort }
   }
 
+  static func getSort(
+    _ child: Relation, _ exprs: [Spark_Connect_Expression], _ isGlobal: Bool = 
true
+  ) -> Plan {
+    var sort = Sort()
+    sort.input = child
+    sort.order = exprs.map {
+      if case .sortOrder(let sortOrder) = $0.exprType {
+        return sortOrder
+      }
+      var sortOrder = Spark_Connect_Expression.SortOrder()
+      sortOrder.child = $0
+      sortOrder.direction = .ascending
+      sortOrder.nullOrdering = .sortNullsFirst
+      return sortOrder
+    }
+    sort.isGlobal = isGlobal
+    return createPlan { $0.sort = sort }
+  }
+
   static func getLimit(_ child: Relation, _ n: Int32) -> Plan {
     var limit = Limit()
     limit.input = child
@@ -1063,6 +1101,18 @@ public actor SparkConnectClient {
     return createPlan { $0.join = join }
   }
 
+  static func getJoin(
+    _ left: Relation, _ right: Relation, _ joinType: JoinType,
+    joinCondition: Spark_Connect_Expression
+  ) -> Plan {
+    var join = Join()
+    join.left = left
+    join.right = right
+    join.joinType = joinType
+    join.joinCondition = joinCondition
+    return createPlan { $0.join = join }
+  }
+
   static func getLateralJoin(
     _ left: Relation, _ right: Relation, _ joinType: JoinType,
     joinCondition: String? = nil
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index a4a4e7b..c49b7dc 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -318,6 +318,19 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func withColumnWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    // Add a new column.
+    #expect(try await spark.range(1).withColumn("b", col("id") + 1).columns == 
["id", "b"])
+    #expect(try await spark.range(2).withColumn("doubled", col("id") * 
2).collect()
+      == [Row(0, 0), Row(1, 2)])
+    // Replace an existing column in place.
+    #expect(try await spark.range(1).withColumn("id", col("id") + 1).columns 
== ["id"])
+    #expect(try await spark.range(1).withColumn("id", col("id") + 1).collect() 
== [Row(1)])
+    await spark.stop()
+  }
+
   @Test
   func drop() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -329,6 +342,17 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func dropWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql("SELECT 1 a, 2 b, 3 c, 4 d")
+    #expect(try await df.drop(col("a")).columns == ["b", "c", "d"])
+    #expect(try await df.drop(col("b"), col("c")).columns == ["a", "d"])
+    // Ignore unknown columns.
+    #expect(try await df.drop(col("x"), col("y")).columns == ["a", "b", "c", 
"d"])
+    await spark.stop()
+  }
+
   @Test
   func filter() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -428,6 +452,20 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func sortWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let expected = Array((1...10).map { Row($0) })
+    #expect(try await spark.range(10, 0, -1).sort(col("id")).collect() == 
expected)
+    #expect(try await spark.range(1, 11).sort(col("id").desc()).collect()
+      == Array(expected.reversed()))
+    // Mix a sort order expression and a plain column.
+    let df = try await spark.sql("SELECT * FROM VALUES (1, 'a'), (2, 'c'), (1, 
'b') T(a, b)")
+    #expect(try await df.sort(col("a").desc(), col("b")).collect()
+      == [Row(2, "c"), Row(1, "a"), Row(1, "b")])
+    await spark.stop()
+  }
+
   @Test
   func orderBy() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -436,6 +474,17 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func orderByWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let expected = Array((1...10).map { Row($0) })
+    #expect(try await spark.range(10, 0, -1).orderBy(col("id")).collect() == 
expected)
+    #expect(try await spark.range(10, 0, 
-1).orderBy(col("id").asc()).collect() == expected)
+    #expect(try await spark.range(1, 11).orderBy(col("id").desc()).collect()
+      == Array(expected.reversed()))
+    await spark.stop()
+  }
+
   @Test
   func sortWithinPartitions() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -446,6 +495,19 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func sortWithinPartitionsWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let expected = Array((1...10).map { Row($0) })
+    #expect(
+      try await spark.range(10, 0, 
-1).repartition(1).sortWithinPartitions(col("id")).collect()
+        == expected)
+    #expect(
+      try await spark.range(1, 
11).repartition(1).sortWithinPartitions(col("id").desc()).collect()
+        == Array(expected.reversed()))
+    await spark.stop()
+  }
+
   @Test
   func table() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -657,6 +719,26 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func joinWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df1 = try await spark.sql("SELECT * FROM VALUES ('a', 1), ('b', 2) AS 
T(a, b)")
+    let df2 = try await spark.sql("SELECT * FROM VALUES ('c', 2), ('d', 3) AS 
S(c, b)")
+    let expected = [Row("b", 2, "c", 2)]
+    #expect(try await df1.join(df2, joinExprs: col("T.b") == 
col("S.b")).collect() == expected)
+    #expect(
+      try await df1.join(df2, joinExprs: col("T.b") == col("S.b"), joinType: 
"inner").collect()
+        == expected)
+    #expect(
+      try await df1.join(df2, joinExprs: col("T.b") == col("S.b"), joinType: 
"left").collect()
+        == [Row("a", 1, nil, nil), Row("b", 2, "c", 2)])
+
+    let df = try await spark.range(3)
+    let joined = try await df.alias("l").join(df.alias("r"), joinExprs: 
col("l.id") == col("r.id"))
+    #expect(try await joined.orderBy(col("l.id")).collect() == [Row(0, 0), 
Row(1, 1), Row(2, 2)])
+    await spark.stop()
+  }
+
   @Test
   func alias() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to