This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new e2c990f [SPARK-58473] Support comparison/logical/arithmetic operators
in `Column`
e2c990f is described below
commit e2c990f8740f918e6ca0cf3d18eb8b5367e21c82
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 08:05:57 2026 -0700
[SPARK-58473] Support comparison/logical/arithmetic operators in `Column`
### What changes were proposed in this pull request?
This PR aims to support comparison, logical, and arithmetic operators in
`Column`.
- Comparison: `==`, `!=`, `<`, `<=`, `>`, `>=`
- Logical: `&&`, `||`, prefix `!`
- Arithmetic: `+`, `-`, `*`, `/`, `%`, prefix `-` (negation)
The operators build `Expression.UnresolvedFunction` with the same function
names as the Scala client (`Column.scala`): `==` → `"="`, `!=` → `"!"` of
`"="`, `&&` → `"and"`, `||` → `"or"`, prefix `-` → `"negative"`, and the same
symbols for the rest.
In addition, `DataFrame.filter(_: Column)` and `DataFrame.where(_: Column)`
overloads are added to make the operators usable.
```swift
let adults = df.filter(col("age") > lit(21) && col("name") == lit("Alice"))
let doubled = df.select((col("id") * lit(2)).alias("doubled"))
```
Note that unlike Scala, Swift allows overloading `==` with a `Column`
return type without conflicts because `Column` does not conform to `Equatable`,
so the Scala-style `===`/`=!=` operators are not needed. Misuse such as `if
col("a") == col("b")` is rejected at compile time (`cannot convert value of
type 'Column' to expected condition type 'Bool'`).
### Why are the changes needed?
To allow users to write filter and projection expressions in the idiomatic
PySpark/Scala style instead of SQL strings.
### Does this PR introduce _any_ user-facing change?
No. This is a new feature addition to the unreleased `Column` API.
### How was this patch tested?
Pass the CIs with the newly added unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #455 from dongjoon-hyun/SPARK-58473.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Column.swift | 99 +++++++++++++++++++
.../SparkConnect/DataFrame+Transformations.swift | 27 ++++++
Sources/SparkConnect/SparkConnectClient.swift | 7 ++
Tests/SparkConnectTests/FunctionsTests.swift | 108 +++++++++++++++++++++
4 files changed, 241 insertions(+)
diff --git a/Sources/SparkConnect/Column.swift
b/Sources/SparkConnect/Column.swift
index 7bb76cc..95be1d2 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -98,4 +98,103 @@ public struct Column: Sendable {
expr.sortOrder = sortOrder
return Column(expr)
}
+
+ private static func fn(_ name: String, _ args: Column...) -> Column {
+ var function = Spark_Connect_Expression.UnresolvedFunction()
+ function.functionName = name
+ function.arguments = args.map { $0.expr }
+ var expr = Spark_Connect_Expression()
+ expr.unresolvedFunction = function
+ return Column(expr)
+ }
+
+ // MARK: - Comparison operators
+
+ /// Returns an equality test expression. Note that this returns a ``Column``
+ /// expression evaluated by Spark, not a `Bool`.
+ ///
+ /// ```swift
+ /// df.filter(col("name") == lit("Alice"))
+ /// ```
+ public static func == (lhs: Column, rhs: Column) -> Column {
+ return fn("=", lhs, rhs)
+ }
+
+ /// Returns an inequality test expression.
+ public static func != (lhs: Column, rhs: Column) -> Column {
+ return fn("!", fn("=", lhs, rhs))
+ }
+
+ /// Returns a less-than test expression.
+ public static func < (lhs: Column, rhs: Column) -> Column {
+ return fn("<", lhs, rhs)
+ }
+
+ /// Returns a less-than-or-equal test expression.
+ public static func <= (lhs: Column, rhs: Column) -> Column {
+ return fn("<=", lhs, rhs)
+ }
+
+ /// Returns a greater-than test expression.
+ public static func > (lhs: Column, rhs: Column) -> Column {
+ return fn(">", lhs, rhs)
+ }
+
+ /// Returns a greater-than-or-equal test expression.
+ public static func >= (lhs: Column, rhs: Column) -> Column {
+ return fn(">=", lhs, rhs)
+ }
+
+ // MARK: - Logical operators
+
+ /// Returns a logical AND expression.
+ ///
+ /// ```swift
+ /// df.filter(col("age") > lit(21) && col("name") == lit("Alice"))
+ /// ```
+ public static func && (lhs: Column, rhs: Column) -> Column {
+ return fn("and", lhs, rhs)
+ }
+
+ /// Returns a logical OR expression.
+ public static func || (lhs: Column, rhs: Column) -> Column {
+ return fn("or", lhs, rhs)
+ }
+
+ /// Returns a logical NOT expression.
+ public static prefix func ! (col: Column) -> Column {
+ return fn("!", col)
+ }
+
+ // MARK: - Arithmetic operators
+
+ /// Returns an addition expression.
+ public static func + (lhs: Column, rhs: Column) -> Column {
+ return fn("+", lhs, rhs)
+ }
+
+ /// Returns a subtraction expression.
+ public static func - (lhs: Column, rhs: Column) -> Column {
+ return fn("-", lhs, rhs)
+ }
+
+ /// Returns a multiplication expression.
+ public static func * (lhs: Column, rhs: Column) -> Column {
+ return fn("*", lhs, rhs)
+ }
+
+ /// Returns a division expression.
+ public static func / (lhs: Column, rhs: Column) -> Column {
+ return fn("/", lhs, rhs)
+ }
+
+ /// Returns a modulo expression.
+ public static func % (lhs: Column, rhs: Column) -> Column {
+ return fn("%", lhs, rhs)
+ }
+
+ /// Returns a negation expression.
+ public static prefix func - (col: Column) -> Column {
+ return fn("negative", col)
+ }
}
diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift
b/Sources/SparkConnect/DataFrame+Transformations.swift
index ecdbe8e..674d223 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -218,6 +218,19 @@ extension DataFrame {
spark: self.spark, plan: SparkConnectClient.getFilter(self.plan.root,
conditionExpr))
}
+ /// Filters rows using the given ``Column`` condition.
+ ///
+ /// ```swift
+ /// let adults = df.filter(col("age") > lit(21) && col("name") ==
lit("Alice"))
+ /// ```
+ ///
+ /// - Parameter condition: A ``Column`` expression for filtering
+ /// - Returns: A new DataFrame containing only rows that match the condition
+ public func filter(_ condition: Column) -> DataFrame {
+ return DataFrame(
+ spark: self.spark, plan: SparkConnectClient.getFilter(self.plan.root,
condition.expr))
+ }
+
/// Filters rows using the given condition (alias for filter).
///
/// This method is an alias for ``filter(_:)`` and behaves identically.
@@ -232,6 +245,20 @@ extension DataFrame {
return filter(conditionExpr)
}
+ /// Filters rows using the given ``Column`` condition (alias for filter).
+ ///
+ /// This method is an alias for ``filter(_:)-(Column)`` and behaves
identically.
+ ///
+ /// ```swift
+ /// let highSalary = df.where(col("salary") > lit(100000))
+ /// ```
+ ///
+ /// - Parameter condition: A ``Column`` expression for filtering
+ /// - Returns: A new DataFrame containing only rows that match the condition
+ public func `where`(_ condition: Column) -> DataFrame {
+ return filter(condition)
+ }
+
/// Returns a new DataFrame sorted by the specified columns.
///
/// By default, sorts in ascending order. Use `desc("column")` for
descending order.
diff --git a/Sources/SparkConnect/SparkConnectClient.swift
b/Sources/SparkConnect/SparkConnectClient.swift
index 2f3d7d7..ff00e27 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -602,6 +602,13 @@ public actor SparkConnectClient {
return createPlan { $0.filter = filter }
}
+ static func getFilter(_ child: Relation, _ condition:
Spark_Connect_Expression) -> Plan {
+ var filter = Filter()
+ filter.input = child
+ filter.condition = condition
+ return createPlan { $0.filter = filter }
+ }
+
static func getDrop(_ child: Relation, _ columnNames: [String]) -> Plan {
var drop = Drop()
drop.input = child
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift
b/Tests/SparkConnectTests/FunctionsTests.swift
index e1440a0..16cee8c 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -107,6 +107,83 @@ struct FunctionsTests {
}
}
+ @Test
+ func comparisonOperators() throws {
+ for (column, name) in [
+ (col("a") == col("b"), "="),
+ (col("a") < col("b"), "<"),
+ (col("a") <= col("b"), "<="),
+ (col("a") > col("b"), ">"),
+ (col("a") >= col("b"), ">="),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+
#expect(expr.unresolvedFunction.arguments[1].unresolvedAttribute.unparsedIdentifier
== "b")
+ }
+ }
+
+ @Test
+ func notEqualOperator() throws {
+ let expr = (col("a") != col("b")).expr
+ #expect(expr.unresolvedFunction.functionName == "!")
+ #expect(expr.unresolvedFunction.arguments.count == 1)
+ let inner = expr.unresolvedFunction.arguments[0].unresolvedFunction
+ #expect(inner.functionName == "=")
+ #expect(inner.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+ #expect(inner.arguments[1].unresolvedAttribute.unparsedIdentifier == "b")
+ }
+
+ @Test
+ func logicalOperators() throws {
+ for (column, name) in [
+ (col("a") && col("b"), "and"),
+ (col("a") || col("b"), "or"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+ }
+
+ let not = (!col("a")).expr
+ #expect(not.unresolvedFunction.functionName == "!")
+ #expect(not.unresolvedFunction.arguments.count == 1)
+
#expect(not.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ }
+
+ @Test
+ func arithmeticOperators() throws {
+ for (column, name) in [
+ (col("a") + col("b"), "+"),
+ (col("a") - col("b"), "-"),
+ (col("a") * col("b"), "*"),
+ (col("a") / col("b"), "/"),
+ (col("a") % col("b"), "%"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+ }
+
+ let negate = (-col("a")).expr
+ #expect(negate.unresolvedFunction.functionName == "negative")
+ #expect(negate.unresolvedFunction.arguments.count == 1)
+
#expect(negate.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ }
+
+ @Test
+ func operatorPrecedence() throws {
+ // Parsed as ((a > b) and (c = d)) or (not e)
+ let expr = (col("a") > col("b") && col("c") == col("d") || !col("e")).expr
+ #expect(expr.unresolvedFunction.functionName == "or")
+ let and = expr.unresolvedFunction.arguments[0].unresolvedFunction
+ #expect(and.functionName == "and")
+ #expect(and.arguments[0].unresolvedFunction.functionName == ">")
+ #expect(and.arguments[1].unresolvedFunction.functionName == "=")
+
#expect(expr.unresolvedFunction.arguments[1].unresolvedFunction.functionName ==
"!")
+ }
+
@Test
func selectColumns() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -125,4 +202,35 @@ struct FunctionsTests {
#expect(rows == [Row(0, 1, 0, 0.0), Row(1, 1, 1, 1.0), Row(2, 1, 2, 2.0)])
await spark.stop()
}
+
+ @Test
+ func filterWithColumn() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(10)
+ #expect(try await df.filter(col("id") == lit(3)).collect() == [Row(3)])
+ #expect(try await df.filter(col("id") != lit(3)).count() == 9)
+ #expect(try await df.filter(col("id") < lit(3)).count() == 3)
+ #expect(try await df.filter(col("id") <= lit(3)).count() == 4)
+ #expect(try await df.filter(col("id") > lit(7)).count() == 2)
+ #expect(try await df.filter(col("id") >= lit(7)).count() == 3)
+ #expect(try await df.filter(col("id") > lit(2) && col("id") <
lit(5)).count() == 2)
+ #expect(try await df.filter(col("id") < lit(2) || col("id") >
lit(7)).count() == 4)
+ #expect(try await df.filter(!(col("id") < lit(8))).count() == 2)
+ #expect(try await df.where(col("id") % lit(3) == lit(0)).count() == 4)
+ await spark.stop()
+ }
+
+ @Test
+ func selectWithOperators() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.range(1, 4)
+ .select(
+ (col("id") + lit(1)).alias("plus"),
+ (col("id") - lit(1)).alias("minus"),
+ (col("id") * lit(2)).alias("times"),
+ (-col("id")).alias("negated")
+ ).orderBy("id").collect()
+ #expect(rows == [Row(2, 0, 2, -1), Row(3, 1, 4, -2), Row(4, 2, 6, -3)])
+ await spark.stop()
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]