This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new cceb2e4 [SPARK-58476] Support Column-based
`sort/withColumn/drop/join` in `DataFrame`
cceb2e4 is described below
commit cceb2e47b37e6219d094ddcea6436dc216707177
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 10:36:55 2026 -0700
[SPARK-58476] Support Column-based `sort/withColumn/drop/join` in
`DataFrame`
### What changes were proposed in this pull request?
This PR aims to support `Column`-based overloads for `sort`, `orderBy`,
`sortWithinPartitions`, `withColumn`, `drop`, and `join` in `DataFrame`.
Previously, only `select`/`agg`/`filter`/`where`/`observe` accepted
`Column` expressions. This PR extends the remaining common entry points to
accept `Column`, matching the Scala client `Dataset` signatures:
- `sort(_ col: Column, _ cols: Column...)`, `orderBy(_ col: Column, _ cols:
Column...)`, `sortWithinPartitions(_ col: Column, _ cols: Column...)` — a
`SortOrder` expression (e.g., `col("id").desc()`) is used as-is; a plain column
is wrapped in ascending order with nulls first, like the Scala client's
`Column.sortOrder`.
- `withColumn(_ colName: String, _ col: Column)` — builds an `Alias`
expression for the `WithColumns` relation.
- `drop(_ col: Column, _ cols: Column...)` — uses the `columns`
(expression) field of the `Drop` relation.
- `join(_ right: DataFrame, joinExprs: Column)` and `join(_ right:
DataFrame, joinExprs: Column, joinType: String)` — passes the column expression
as the `join_condition`.
All existing `String`-based overloads are kept intact.
### Why are the changes needed?
To improve the usability of the `Column` API (`col`, `lit`, operators,
`asc`/`desc`) by allowing it in the main `DataFrame` transformation entry
points, for feature parity with the Scala/PySpark clients.
```swift
df.sort(col("department"), col("salary").desc())
df.withColumn("doubled", col("id") * 2)
df.drop(col("a"), col("b"))
df1.alias("l").join(df2.alias("r"), joinExprs: col("l.id") == col("r.id"),
joinType: "left")
```
### Does this PR introduce _any_ user-facing change?
Yes, this adds new APIs. The existing string-based APIs are unchanged.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #458 from dongjoon-hyun/SPARK-58476.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../SparkConnect/DataFrame+Transformations.swift | 108 +++++++++++++++++++++
Sources/SparkConnect/SparkConnectClient.swift | 50 ++++++++++
Tests/SparkConnectTests/DataFrameTests.swift | 82 ++++++++++++++++
3 files changed, 240 insertions(+)
diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift
b/Sources/SparkConnect/DataFrame+Transformations.swift
index 674d223..0134181 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -109,6 +109,21 @@ extension DataFrame {
return DataFrame(spark: self.spark, plan:
SparkConnectClient.getDrop(self.plan.root, cols))
}
+ /// Returns a new Dataset with columns dropped.
+ /// This is a no-op if schema doesn't contain the column expressions.
+ ///
+ /// ```swift
+ /// let df2 = df.drop(col("a"), col("b"))
+ /// ```
+ /// - Parameters:
+ /// - col: A ``Column`` to drop.
+ /// - cols: Additional ``Column``s to drop.
+ /// - Returns: A ``DataFrame`` without the given columns.
+ public func drop(_ col: Column, _ cols: Column...) -> DataFrame {
+ let exprs = ([col] + cols).map { $0.expr }
+ return DataFrame(spark: self.spark, plan:
SparkConnectClient.getDrop(self.plan.root, exprs))
+ }
+
/// Returns a new ``DataFrame`` that contains only the unique rows from this
``DataFrame``.
/// This is an alias for `distinct`. If column names are given, Spark
considers only those columns.
/// - Parameter cols: Column names
@@ -185,6 +200,22 @@ extension DataFrame {
return withColumns([colName: expr])
}
+ /// Returns a new ``DataFrame`` by adding a column or replacing the existing
column that has the
+ /// same name.
+ ///
+ /// ```swift
+ /// let df2 = df.withColumn("doubled", col("id") * 2)
+ /// ```
+ /// - Parameters:
+ /// - colName: A new column name.
+ /// - col: A ``Column`` expression for the new column.
+ /// - Returns: A ``DataFrame`` with the new or replaced column.
+ public func withColumn(_ colName: String, _ col: Column) -> DataFrame {
+ return DataFrame(
+ spark: self.spark,
+ plan: SparkConnectClient.getWithColumns(self.plan.root, colName,
col.expr))
+ }
+
/// Returns a new ``DataFrame`` by adding columns or replacing the existing
columns that have the
/// same names.
/// - Parameter colsMap: A dictionary of column name and SQL expression
string.
@@ -280,6 +311,21 @@ extension DataFrame {
return DataFrame(spark: self.spark, plan:
SparkConnectClient.getSort(self.plan.root, cols))
}
+ /// Returns a new ``DataFrame`` sorted by the specified ``Column``
expressions.
+ /// Columns without an explicit sort order are sorted in ascending order.
+ ///
+ /// ```swift
+ /// let sorted = df.sort(col("department"), col("salary").desc())
+ /// ```
+ /// - Parameters:
+ /// - col: A ``Column`` or sort order expression to sort by.
+ /// - cols: Additional ``Column`` or sort order expressions.
+ /// - Returns: A new DataFrame sorted by the specified columns
+ public func sort(_ col: Column, _ cols: Column...) -> DataFrame {
+ let exprs = ([col] + cols).map { $0.expr }
+ return DataFrame(spark: self.spark, plan:
SparkConnectClient.getSort(self.plan.root, exprs))
+ }
+
/// Return a new ``DataFrame`` sorted by the specified column(s).
/// - Parameter cols: Column names.
/// - Returns: A sorted ``DataFrame``
@@ -287,6 +333,21 @@ extension DataFrame {
return DataFrame(spark: self.spark, plan:
SparkConnectClient.getSort(self.plan.root, cols))
}
+ /// Return a new ``DataFrame`` sorted by the specified ``Column``
expressions.
+ /// Columns without an explicit sort order are sorted in ascending order.
+ ///
+ /// ```swift
+ /// let sorted = df.orderBy(col("id").desc())
+ /// ```
+ /// - Parameters:
+ /// - col: A ``Column`` or sort order expression to sort by.
+ /// - cols: Additional ``Column`` or sort order expressions.
+ /// - Returns: A sorted ``DataFrame``
+ public func orderBy(_ col: Column, _ cols: Column...) -> DataFrame {
+ let exprs = ([col] + cols).map { $0.expr }
+ return DataFrame(spark: self.spark, plan:
SparkConnectClient.getSort(self.plan.root, exprs))
+ }
+
/// Returns a new ``DataFrame`` with each partition sorted by the specified
column(s).
/// This is the same operation as `SORT BY` in SQL (Hive QL).
/// - Parameter cols: Column names.
@@ -296,6 +357,23 @@ extension DataFrame {
spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root,
cols, false))
}
+ /// Returns a new ``DataFrame`` with each partition sorted by the specified
``Column``
+ /// expressions. Columns without an explicit sort order are sorted in
ascending order.
+ /// This is the same operation as `SORT BY` in SQL (Hive QL).
+ ///
+ /// ```swift
+ /// let sorted = df.sortWithinPartitions(col("id").desc())
+ /// ```
+ /// - Parameters:
+ /// - col: A ``Column`` or sort order expression to sort by.
+ /// - cols: Additional ``Column`` or sort order expressions.
+ /// - Returns: A ``DataFrame`` with each partition sorted.
+ public func sortWithinPartitions(_ col: Column, _ cols: Column...) ->
DataFrame {
+ let exprs = ([col] + cols).map { $0.expr }
+ return DataFrame(
+ spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root,
exprs, false))
+ }
+
/// Limits the result count to the number specified.
///
/// This transformation is often used for:
@@ -505,6 +583,36 @@ extension DataFrame {
return DataFrame(spark: self.spark, plan: plan)
}
+ /// Inner join with another `DataFrame`, using the given join expression.
+ ///
+ /// ```swift
+ /// let joined = df1.alias("a").join(df2.alias("b"), joinExprs: col("a.id")
== col("b.id"))
+ /// ```
+ /// - Parameters:
+ /// - right: Right side of the join operation.
+ /// - joinExprs: A join expression ``Column``.
+ /// - Returns: A `DataFrame`.
+ public func join(_ right: DataFrame, joinExprs: Column) async -> DataFrame {
+ return await join(right, joinExprs: joinExprs, joinType: "inner")
+ }
+
+ /// Join with another `DataFrame`, using the given join expression and join
type.
+ /// - Parameters:
+ /// - right: Right side of the join operation.
+ /// - joinExprs: A join expression ``Column``.
+ /// - joinType: A join type name.
+ /// - Returns: A `DataFrame`.
+ public func join(_ right: DataFrame, joinExprs: Column, joinType: String)
async -> DataFrame {
+ let rightPlan = await (right.getPlan() as! Plan).root
+ let plan = SparkConnectClient.getJoin(
+ self.plan.root,
+ rightPlan,
+ joinType.toJoinType,
+ joinCondition: joinExprs.expr
+ )
+ return DataFrame(spark: self.spark, plan: plan)
+ }
+
/// Explicit cartesian join with another `DataFrame`.
/// - Parameter right: Right side of the join operation.
/// - Returns: A `DataFrame`.
diff --git a/Sources/SparkConnect/SparkConnectClient.swift
b/Sources/SparkConnect/SparkConnectClient.swift
index ff00e27..e545766 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -595,6 +595,18 @@ public actor SparkConnectClient {
return createPlan { $0.withColumns = withColumns }
}
+ static func getWithColumns(
+ _ child: Relation, _ colName: String, _ expr: Spark_Connect_Expression
+ ) -> Plan {
+ var withColumns = WithColumns()
+ withColumns.input = child
+ var alias = Spark_Connect_Expression.Alias()
+ alias.expr = expr
+ alias.name = [colName]
+ withColumns.aliases = [alias]
+ return createPlan { $0.withColumns = withColumns }
+ }
+
static func getFilter(_ child: Relation, _ conditionExpr: String) -> Plan {
var filter = Filter()
filter.input = child
@@ -616,6 +628,13 @@ public actor SparkConnectClient {
return createPlan { $0.drop = drop }
}
+ static func getDrop(_ child: Relation, _ columns:
[Spark_Connect_Expression]) -> Plan {
+ var drop = Drop()
+ drop.input = child
+ drop.columns = columns
+ return createPlan { $0.drop = drop }
+ }
+
static func getDropDuplicates(
_ child: Relation,
_ columnNames: [String],
@@ -723,6 +742,25 @@ public actor SparkConnectClient {
return createPlan { $0.sort = sort }
}
+ static func getSort(
+ _ child: Relation, _ exprs: [Spark_Connect_Expression], _ isGlobal: Bool =
true
+ ) -> Plan {
+ var sort = Sort()
+ sort.input = child
+ sort.order = exprs.map {
+ if case .sortOrder(let sortOrder) = $0.exprType {
+ return sortOrder
+ }
+ var sortOrder = Spark_Connect_Expression.SortOrder()
+ sortOrder.child = $0
+ sortOrder.direction = .ascending
+ sortOrder.nullOrdering = .sortNullsFirst
+ return sortOrder
+ }
+ sort.isGlobal = isGlobal
+ return createPlan { $0.sort = sort }
+ }
+
static func getLimit(_ child: Relation, _ n: Int32) -> Plan {
var limit = Limit()
limit.input = child
@@ -1063,6 +1101,18 @@ public actor SparkConnectClient {
return createPlan { $0.join = join }
}
+ static func getJoin(
+ _ left: Relation, _ right: Relation, _ joinType: JoinType,
+ joinCondition: Spark_Connect_Expression
+ ) -> Plan {
+ var join = Join()
+ join.left = left
+ join.right = right
+ join.joinType = joinType
+ join.joinCondition = joinCondition
+ return createPlan { $0.join = join }
+ }
+
static func getLateralJoin(
_ left: Relation, _ right: Relation, _ joinType: JoinType,
joinCondition: String? = nil
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index a4a4e7b..c49b7dc 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -318,6 +318,19 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func withColumnWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ // Add a new column.
+ #expect(try await spark.range(1).withColumn("b", col("id") + 1).columns ==
["id", "b"])
+ #expect(try await spark.range(2).withColumn("doubled", col("id") *
2).collect()
+ == [Row(0, 0), Row(1, 2)])
+ // Replace an existing column in place.
+ #expect(try await spark.range(1).withColumn("id", col("id") + 1).columns
== ["id"])
+ #expect(try await spark.range(1).withColumn("id", col("id") + 1).collect()
== [Row(1)])
+ await spark.stop()
+ }
+
@Test
func drop() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -329,6 +342,17 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func dropWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql("SELECT 1 a, 2 b, 3 c, 4 d")
+ #expect(try await df.drop(col("a")).columns == ["b", "c", "d"])
+ #expect(try await df.drop(col("b"), col("c")).columns == ["a", "d"])
+ // Ignore unknown columns.
+ #expect(try await df.drop(col("x"), col("y")).columns == ["a", "b", "c",
"d"])
+ await spark.stop()
+ }
+
@Test
func filter() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -428,6 +452,20 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func sortWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let expected = Array((1...10).map { Row($0) })
+ #expect(try await spark.range(10, 0, -1).sort(col("id")).collect() ==
expected)
+ #expect(try await spark.range(1, 11).sort(col("id").desc()).collect()
+ == Array(expected.reversed()))
+ // Mix a sort order expression and a plain column.
+ let df = try await spark.sql("SELECT * FROM VALUES (1, 'a'), (2, 'c'), (1,
'b') T(a, b)")
+ #expect(try await df.sort(col("a").desc(), col("b")).collect()
+ == [Row(2, "c"), Row(1, "a"), Row(1, "b")])
+ await spark.stop()
+ }
+
@Test
func orderBy() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -436,6 +474,17 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func orderByWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let expected = Array((1...10).map { Row($0) })
+ #expect(try await spark.range(10, 0, -1).orderBy(col("id")).collect() ==
expected)
+ #expect(try await spark.range(10, 0,
-1).orderBy(col("id").asc()).collect() == expected)
+ #expect(try await spark.range(1, 11).orderBy(col("id").desc()).collect()
+ == Array(expected.reversed()))
+ await spark.stop()
+ }
+
@Test
func sortWithinPartitions() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -446,6 +495,19 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func sortWithinPartitionsWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let expected = Array((1...10).map { Row($0) })
+ #expect(
+ try await spark.range(10, 0,
-1).repartition(1).sortWithinPartitions(col("id")).collect()
+ == expected)
+ #expect(
+ try await spark.range(1,
11).repartition(1).sortWithinPartitions(col("id").desc()).collect()
+ == Array(expected.reversed()))
+ await spark.stop()
+ }
+
@Test
func table() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -657,6 +719,26 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func joinWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df1 = try await spark.sql("SELECT * FROM VALUES ('a', 1), ('b', 2) AS
T(a, b)")
+ let df2 = try await spark.sql("SELECT * FROM VALUES ('c', 2), ('d', 3) AS
S(c, b)")
+ let expected = [Row("b", 2, "c", 2)]
+ #expect(try await df1.join(df2, joinExprs: col("T.b") ==
col("S.b")).collect() == expected)
+ #expect(
+ try await df1.join(df2, joinExprs: col("T.b") == col("S.b"), joinType:
"inner").collect()
+ == expected)
+ #expect(
+ try await df1.join(df2, joinExprs: col("T.b") == col("S.b"), joinType:
"left").collect()
+ == [Row("a", 1, nil, nil), Row("b", 2, "c", 2)])
+
+ let df = try await spark.range(3)
+ let joined = try await df.alias("l").join(df.alias("r"), joinExprs:
col("l.id") == col("r.id"))
+ #expect(try await joined.orderBy(col("l.id")).collect() == [Row(0, 0),
Row(1, 1), Row(2, 2)])
+ await spark.stop()
+ }
+
@Test
func alias() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]