This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new e2c990f  [SPARK-58473] Support comparison/logical/arithmetic operators 
in `Column`
e2c990f is described below

commit e2c990f8740f918e6ca0cf3d18eb8b5367e21c82
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 08:05:57 2026 -0700

    [SPARK-58473] Support comparison/logical/arithmetic operators in `Column`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support comparison, logical, and arithmetic operators in 
`Column`.
    
    - Comparison: `==`, `!=`, `<`, `<=`, `>`, `>=`
    - Logical: `&&`, `||`, prefix `!`
    - Arithmetic: `+`, `-`, `*`, `/`, `%`, prefix `-` (negation)
    
    The operators build `Expression.UnresolvedFunction` with the same function 
names as the Scala client (`Column.scala`): `==` → `"="`, `!=` → `"!"` of 
`"="`, `&&` → `"and"`, `||` → `"or"`, prefix `-` → `"negative"`, and the same 
symbols for the rest.
    
    In addition, `DataFrame.filter(_: Column)` and `DataFrame.where(_: Column)` 
overloads are added to make the operators usable.
    
    ```swift
    let adults = df.filter(col("age") > lit(21) && col("name") == lit("Alice"))
    let doubled = df.select((col("id") * lit(2)).alias("doubled"))
    ```
    
    Note that unlike Scala, Swift allows overloading `==` with a `Column` 
return type without conflicts because `Column` does not conform to `Equatable`, 
so the Scala-style `===`/`=!=` operators are not needed. Misuse such as `if 
col("a") == col("b")` is rejected at compile time (`cannot convert value of 
type 'Column' to expected condition type 'Bool'`).
    
    ### Why are the changes needed?
    
    To allow users to write filter and projection expressions in the idiomatic 
PySpark/Scala style instead of SQL strings.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a new feature addition to the unreleased `Column` API.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added unit tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #455 from dongjoon-hyun/SPARK-58473.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/Column.swift                  |  99 +++++++++++++++++++
 .../SparkConnect/DataFrame+Transformations.swift   |  27 ++++++
 Sources/SparkConnect/SparkConnectClient.swift      |   7 ++
 Tests/SparkConnectTests/FunctionsTests.swift       | 108 +++++++++++++++++++++
 4 files changed, 241 insertions(+)

diff --git a/Sources/SparkConnect/Column.swift 
b/Sources/SparkConnect/Column.swift
index 7bb76cc..95be1d2 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -98,4 +98,103 @@ public struct Column: Sendable {
     expr.sortOrder = sortOrder
     return Column(expr)
   }
+
+  private static func fn(_ name: String, _ args: Column...) -> Column {
+    var function = Spark_Connect_Expression.UnresolvedFunction()
+    function.functionName = name
+    function.arguments = args.map { $0.expr }
+    var expr = Spark_Connect_Expression()
+    expr.unresolvedFunction = function
+    return Column(expr)
+  }
+
+  // MARK: - Comparison operators
+
+  /// Returns an equality test expression. Note that this returns a ``Column``
+  /// expression evaluated by Spark, not a `Bool`.
+  ///
+  /// ```swift
+  /// df.filter(col("name") == lit("Alice"))
+  /// ```
+  public static func == (lhs: Column, rhs: Column) -> Column {
+    return fn("=", lhs, rhs)
+  }
+
+  /// Returns an inequality test expression.
+  public static func != (lhs: Column, rhs: Column) -> Column {
+    return fn("!", fn("=", lhs, rhs))
+  }
+
+  /// Returns a less-than test expression.
+  public static func < (lhs: Column, rhs: Column) -> Column {
+    return fn("<", lhs, rhs)
+  }
+
+  /// Returns a less-than-or-equal test expression.
+  public static func <= (lhs: Column, rhs: Column) -> Column {
+    return fn("<=", lhs, rhs)
+  }
+
+  /// Returns a greater-than test expression.
+  public static func > (lhs: Column, rhs: Column) -> Column {
+    return fn(">", lhs, rhs)
+  }
+
+  /// Returns a greater-than-or-equal test expression.
+  public static func >= (lhs: Column, rhs: Column) -> Column {
+    return fn(">=", lhs, rhs)
+  }
+
+  // MARK: - Logical operators
+
+  /// Returns a logical AND expression.
+  ///
+  /// ```swift
+  /// df.filter(col("age") > lit(21) && col("name") == lit("Alice"))
+  /// ```
+  public static func && (lhs: Column, rhs: Column) -> Column {
+    return fn("and", lhs, rhs)
+  }
+
+  /// Returns a logical OR expression.
+  public static func || (lhs: Column, rhs: Column) -> Column {
+    return fn("or", lhs, rhs)
+  }
+
+  /// Returns a logical NOT expression.
+  public static prefix func ! (col: Column) -> Column {
+    return fn("!", col)
+  }
+
+  // MARK: - Arithmetic operators
+
+  /// Returns an addition expression.
+  public static func + (lhs: Column, rhs: Column) -> Column {
+    return fn("+", lhs, rhs)
+  }
+
+  /// Returns a subtraction expression.
+  public static func - (lhs: Column, rhs: Column) -> Column {
+    return fn("-", lhs, rhs)
+  }
+
+  /// Returns a multiplication expression.
+  public static func * (lhs: Column, rhs: Column) -> Column {
+    return fn("*", lhs, rhs)
+  }
+
+  /// Returns a division expression.
+  public static func / (lhs: Column, rhs: Column) -> Column {
+    return fn("/", lhs, rhs)
+  }
+
+  /// Returns a modulo expression.
+  public static func % (lhs: Column, rhs: Column) -> Column {
+    return fn("%", lhs, rhs)
+  }
+
+  /// Returns a negation expression.
+  public static prefix func - (col: Column) -> Column {
+    return fn("negative", col)
+  }
 }
diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift 
b/Sources/SparkConnect/DataFrame+Transformations.swift
index ecdbe8e..674d223 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -218,6 +218,19 @@ extension DataFrame {
       spark: self.spark, plan: SparkConnectClient.getFilter(self.plan.root, 
conditionExpr))
   }
 
+  /// Filters rows using the given ``Column`` condition.
+  ///
+  /// ```swift
+  /// let adults = df.filter(col("age") > lit(21) && col("name") == 
lit("Alice"))
+  /// ```
+  ///
+  /// - Parameter condition: A ``Column`` expression for filtering
+  /// - Returns: A new DataFrame containing only rows that match the condition
+  public func filter(_ condition: Column) -> DataFrame {
+    return DataFrame(
+      spark: self.spark, plan: SparkConnectClient.getFilter(self.plan.root, 
condition.expr))
+  }
+
   /// Filters rows using the given condition (alias for filter).
   ///
   /// This method is an alias for ``filter(_:)`` and behaves identically.
@@ -232,6 +245,20 @@ extension DataFrame {
     return filter(conditionExpr)
   }
 
+  /// Filters rows using the given ``Column`` condition (alias for filter).
+  ///
+  /// This method is an alias for ``filter(_:)-(Column)`` and behaves 
identically.
+  ///
+  /// ```swift
+  /// let highSalary = df.where(col("salary") > lit(100000))
+  /// ```
+  ///
+  /// - Parameter condition: A ``Column`` expression for filtering
+  /// - Returns: A new DataFrame containing only rows that match the condition
+  public func `where`(_ condition: Column) -> DataFrame {
+    return filter(condition)
+  }
+
   /// Returns a new DataFrame sorted by the specified columns.
   ///
   /// By default, sorts in ascending order. Use `desc("column")` for 
descending order.
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index 2f3d7d7..ff00e27 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -602,6 +602,13 @@ public actor SparkConnectClient {
     return createPlan { $0.filter = filter }
   }
 
+  static func getFilter(_ child: Relation, _ condition: 
Spark_Connect_Expression) -> Plan {
+    var filter = Filter()
+    filter.input = child
+    filter.condition = condition
+    return createPlan { $0.filter = filter }
+  }
+
   static func getDrop(_ child: Relation, _ columnNames: [String]) -> Plan {
     var drop = Drop()
     drop.input = child
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift 
b/Tests/SparkConnectTests/FunctionsTests.swift
index e1440a0..16cee8c 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -107,6 +107,83 @@ struct FunctionsTests {
     }
   }
 
+  @Test
+  func comparisonOperators() throws {
+    for (column, name) in [
+      (col("a") == col("b"), "="),
+      (col("a") < col("b"), "<"),
+      (col("a") <= col("b"), "<="),
+      (col("a") > col("b"), ">"),
+      (col("a") >= col("b"), ">="),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments.count == 2)
+      
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+      
#expect(expr.unresolvedFunction.arguments[1].unresolvedAttribute.unparsedIdentifier
 == "b")
+    }
+  }
+
+  @Test
+  func notEqualOperator() throws {
+    let expr = (col("a") != col("b")).expr
+    #expect(expr.unresolvedFunction.functionName == "!")
+    #expect(expr.unresolvedFunction.arguments.count == 1)
+    let inner = expr.unresolvedFunction.arguments[0].unresolvedFunction
+    #expect(inner.functionName == "=")
+    #expect(inner.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+    #expect(inner.arguments[1].unresolvedAttribute.unparsedIdentifier == "b")
+  }
+
+  @Test
+  func logicalOperators() throws {
+    for (column, name) in [
+      (col("a") && col("b"), "and"),
+      (col("a") || col("b"), "or"),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments.count == 2)
+    }
+
+    let not = (!col("a")).expr
+    #expect(not.unresolvedFunction.functionName == "!")
+    #expect(not.unresolvedFunction.arguments.count == 1)
+    
#expect(not.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+  }
+
+  @Test
+  func arithmeticOperators() throws {
+    for (column, name) in [
+      (col("a") + col("b"), "+"),
+      (col("a") - col("b"), "-"),
+      (col("a") * col("b"), "*"),
+      (col("a") / col("b"), "/"),
+      (col("a") % col("b"), "%"),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments.count == 2)
+    }
+
+    let negate = (-col("a")).expr
+    #expect(negate.unresolvedFunction.functionName == "negative")
+    #expect(negate.unresolvedFunction.arguments.count == 1)
+    
#expect(negate.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+  }
+
+  @Test
+  func operatorPrecedence() throws {
+    // Parsed as ((a > b) and (c = d)) or (not e)
+    let expr = (col("a") > col("b") && col("c") == col("d") || !col("e")).expr
+    #expect(expr.unresolvedFunction.functionName == "or")
+    let and = expr.unresolvedFunction.arguments[0].unresolvedFunction
+    #expect(and.functionName == "and")
+    #expect(and.arguments[0].unresolvedFunction.functionName == ">")
+    #expect(and.arguments[1].unresolvedFunction.functionName == "=")
+    
#expect(expr.unresolvedFunction.arguments[1].unresolvedFunction.functionName == 
"!")
+  }
+
   @Test
   func selectColumns() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -125,4 +202,35 @@ struct FunctionsTests {
     #expect(rows == [Row(0, 1, 0, 0.0), Row(1, 1, 1, 1.0), Row(2, 1, 2, 2.0)])
     await spark.stop()
   }
+
+  @Test
+  func filterWithColumn() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.range(10)
+    #expect(try await df.filter(col("id") == lit(3)).collect() == [Row(3)])
+    #expect(try await df.filter(col("id") != lit(3)).count() == 9)
+    #expect(try await df.filter(col("id") < lit(3)).count() == 3)
+    #expect(try await df.filter(col("id") <= lit(3)).count() == 4)
+    #expect(try await df.filter(col("id") > lit(7)).count() == 2)
+    #expect(try await df.filter(col("id") >= lit(7)).count() == 3)
+    #expect(try await df.filter(col("id") > lit(2) && col("id") < 
lit(5)).count() == 2)
+    #expect(try await df.filter(col("id") < lit(2) || col("id") > 
lit(7)).count() == 4)
+    #expect(try await df.filter(!(col("id") < lit(8))).count() == 2)
+    #expect(try await df.where(col("id") % lit(3) == lit(0)).count() == 4)
+    await spark.stop()
+  }
+
+  @Test
+  func selectWithOperators() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await spark.range(1, 4)
+      .select(
+        (col("id") + lit(1)).alias("plus"),
+        (col("id") - lit(1)).alias("minus"),
+        (col("id") * lit(2)).alias("times"),
+        (-col("id")).alias("negated")
+      ).orderBy("id").collect()
+    #expect(rows == [Row(2, 0, 2, -1), Row(3, 1, 4, -2), Row(4, 2, 6, -3)])
+    await spark.stop()
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to