This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new c7ace77  [SPARK-58475] Support predicate and string methods in `Column`
c7ace77 is described below

commit c7ace776868bd19fe05b3e5ed037d49a1aab5e0f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 10:36:07 2026 -0700

    [SPARK-58475] Support predicate and string methods in `Column`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support predicate, string, and extraction methods in 
`Column`.
    
    - Predicates: `isNull`, `isNotNull`, `isin`, `between`, `eqNullSafe`
    - String methods: `like`, `rlike`, `ilike`, `contains`, `startsWith`, 
`endsWith`, `substr`
    - Extraction: `getItem`, `getField`
    
    The function name mappings follow the Scala client's `Column.scala`: 
`isnull`/`isnotnull`, `in`, `<=>`, `like`/`rlike`/`ilike`, 
`contains`/`startswith`/`endswith`, `substr`. `between` is composed as `(self 
>= lower) && (self <= upper)` like the Scala client, and `getItem`/`getField` 
use `Expression.UnresolvedExtractValue` like the Scala client's 
`apply(extraction)`. Methods taking operands accept both `Column` and 
`SparkLiteral` values.
    
    ### Why are the changes needed?
    
    To allow Swift users to build filter and projection expressions 
idiomatically without falling back to SQL strings.
    
    ```swift
    df.filter(col("name").isNotNull() && col("age").between(20, 30))
    df.filter(col("name").like("Al%"))
    df.select(col("arr").getItem(0), col("s").getField("a"))
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a new API addition.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added unit tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #457 from dongjoon-hyun/SPARK-58475.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/Column.swift            | 226 +++++++++++++++++++++++++++
 Tests/SparkConnectTests/FunctionsTests.swift | 121 ++++++++++++++
 2 files changed, 347 insertions(+)

diff --git a/Sources/SparkConnect/Column.swift 
b/Sources/SparkConnect/Column.swift
index b842663..d7663f0 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -86,6 +86,228 @@ public struct Column: Sendable {
     return Column(expr)
   }
 
+  // MARK: - Predicates
+
+  /// Returns an expression that is true if this column is null.
+  ///
+  /// ```swift
+  /// df.filter(col("name").isNull())
+  /// ```
+  /// - Returns: A ``Column`` testing for null.
+  public func isNull() -> Column {
+    return Column.fn("isnull", self)
+  }
+
+  /// Returns an expression that is true if this column is not null.
+  ///
+  /// ```swift
+  /// df.filter(col("name").isNotNull())
+  /// ```
+  /// - Returns: A ``Column`` testing for non-null.
+  public func isNotNull() -> Column {
+    return Column.fn("isnotnull", self)
+  }
+
+  /// Returns an expression that is true if this column's value is contained in
+  /// the given values.
+  ///
+  /// ```swift
+  /// df.filter(col("age").isin(20, 30))
+  /// ```
+  /// - Parameter values: Literal values to test against.
+  /// - Returns: A ``Column`` testing for membership.
+  public func isin(_ values: any SparkLiteral...) -> Column {
+    return Column.fn("in", [self] + values.map { $0.toLiteralColumn })
+  }
+
+  /// Returns an expression that is true if this column is between the given
+  /// lower and upper bounds, inclusive.
+  ///
+  /// ```swift
+  /// df.filter(col("age").between(20, 30))
+  /// ```
+  /// - Parameters:
+  ///   - lower: A lower bound.
+  ///   - upper: An upper bound.
+  /// - Returns: A ``Column`` testing for the range.
+  public func between(_ lower: Column, _ upper: Column) -> Column {
+    return (self >= lower) && (self <= upper)
+  }
+
+  /// Returns an expression that is true if this column is between the given
+  /// lower and upper bounds, inclusive.
+  public func between(_ lower: Column, _ upper: some SparkLiteral) -> Column {
+    return (self >= lower) && (self <= upper)
+  }
+
+  /// Returns an expression that is true if this column is between the given
+  /// lower and upper bounds, inclusive.
+  public func between(_ lower: some SparkLiteral, _ upper: Column) -> Column {
+    return (self >= lower) && (self <= upper)
+  }
+
+  /// Returns an expression that is true if this column is between the given
+  /// lower and upper bounds, inclusive.
+  public func between(_ lower: some SparkLiteral, _ upper: some SparkLiteral) 
-> Column {
+    return (self >= lower) && (self <= upper)
+  }
+
+  /// Returns a null-safe equality test expression. Unlike `==`, this returns
+  /// `true` when both sides are null.
+  ///
+  /// ```swift
+  /// df.filter(col("name").eqNullSafe(col("nickname")))
+  /// ```
+  /// - Parameter other: A ``Column`` to compare with.
+  /// - Returns: A ``Column`` testing for null-safe equality.
+  public func eqNullSafe(_ other: Column) -> Column {
+    return Column.fn("<=>", self, other)
+  }
+
+  /// Returns a null-safe equality test expression against a literal value.
+  public func eqNullSafe(_ other: some SparkLiteral) -> Column {
+    return eqNullSafe(other.toLiteralColumn)
+  }
+
+  // MARK: - String methods
+
+  /// Returns a SQL `LIKE` expression.
+  ///
+  /// ```swift
+  /// df.filter(col("name").like("Al%"))
+  /// ```
+  /// - Parameter pattern: A SQL LIKE pattern.
+  /// - Returns: A ``Column`` testing for the pattern match.
+  public func like(_ pattern: String) -> Column {
+    return Column.fn("like", self, pattern.toLiteralColumn)
+  }
+
+  /// Returns a SQL `RLIKE` expression (LIKE with regex).
+  ///
+  /// ```swift
+  /// df.filter(col("name").rlike("^Al.*"))
+  /// ```
+  /// - Parameter pattern: A regular expression.
+  /// - Returns: A ``Column`` testing for the regex match.
+  public func rlike(_ pattern: String) -> Column {
+    return Column.fn("rlike", self, pattern.toLiteralColumn)
+  }
+
+  /// Returns a SQL `ILIKE` expression (case insensitive LIKE).
+  ///
+  /// ```swift
+  /// df.filter(col("name").ilike("al%"))
+  /// ```
+  /// - Parameter pattern: A SQL LIKE pattern.
+  /// - Returns: A ``Column`` testing for the case-insensitive pattern match.
+  public func ilike(_ pattern: String) -> Column {
+    return Column.fn("ilike", self, pattern.toLiteralColumn)
+  }
+
+  /// Returns an expression that is true if this column contains the other 
value.
+  ///
+  /// ```swift
+  /// df.filter(col("name").contains("li"))
+  /// ```
+  /// - Parameter other: A ``Column`` to search for.
+  /// - Returns: A ``Column`` testing for containment.
+  public func contains(_ other: Column) -> Column {
+    return Column.fn("contains", self, other)
+  }
+
+  /// Returns an expression that is true if this column contains the literal 
value.
+  public func contains(_ other: some SparkLiteral) -> Column {
+    return contains(other.toLiteralColumn)
+  }
+
+  /// Returns an expression that is true if this column starts with the other 
value.
+  ///
+  /// ```swift
+  /// df.filter(col("name").startsWith("Al"))
+  /// ```
+  /// - Parameter other: A ``Column`` prefix.
+  /// - Returns: A ``Column`` testing for the prefix match.
+  public func startsWith(_ other: Column) -> Column {
+    return Column.fn("startswith", self, other)
+  }
+
+  /// Returns an expression that is true if this column starts with the 
literal string.
+  public func startsWith(_ literal: String) -> Column {
+    return startsWith(literal.toLiteralColumn)
+  }
+
+  /// Returns an expression that is true if this column ends with the other 
value.
+  ///
+  /// ```swift
+  /// df.filter(col("name").endsWith("ce"))
+  /// ```
+  /// - Parameter other: A ``Column`` suffix.
+  /// - Returns: A ``Column`` testing for the suffix match.
+  public func endsWith(_ other: Column) -> Column {
+    return Column.fn("endswith", self, other)
+  }
+
+  /// Returns an expression that is true if this column ends with the literal 
string.
+  public func endsWith(_ literal: String) -> Column {
+    return endsWith(literal.toLiteralColumn)
+  }
+
+  /// Returns a substring expression.
+  ///
+  /// ```swift
+  /// df.select(col("name").substr(lit(1), lit(3)))
+  /// ```
+  /// - Parameters:
+  ///   - startPos: A starting position (1-based).
+  ///   - len: A substring length.
+  /// - Returns: A ``Column`` of the substring.
+  public func substr(_ startPos: Column, _ len: Column) -> Column {
+    return Column.fn("substr", self, startPos, len)
+  }
+
+  /// Returns a substring expression.
+  ///
+  /// ```swift
+  /// df.select(col("name").substr(1, 3))
+  /// ```
+  public func substr(_ startPos: Int, _ len: Int) -> Column {
+    return substr(startPos.toLiteralColumn, len.toLiteralColumn)
+  }
+
+  // MARK: - Extraction
+
+  /// Returns an expression that gets an item at the given position out of an
+  /// array, or gets a value by the given key out of a map.
+  ///
+  /// ```swift
+  /// df.select(col("arr").getItem(0), col("map").getItem("key"))
+  /// ```
+  /// - Parameter key: An array position (0-based) or a map key.
+  /// - Returns: A ``Column`` of the extracted value.
+  public func getItem(_ key: some SparkLiteral) -> Column {
+    return extractValue(key.toLiteralColumn)
+  }
+
+  /// Returns an expression that gets a field by the given name in a struct.
+  ///
+  /// ```swift
+  /// df.select(col("struct").getField("a"))
+  /// ```
+  /// - Parameter name: A field name.
+  /// - Returns: A ``Column`` of the extracted field.
+  public func getField(_ name: String) -> Column {
+    return extractValue(name.toLiteralColumn)
+  }
+
+  private func extractValue(_ extraction: Column) -> Column {
+    var extractValue = Spark_Connect_Expression.UnresolvedExtractValue()
+    extractValue.child = self.expr
+    extractValue.extraction = extraction.expr
+    var expr = Spark_Connect_Expression()
+    expr.unresolvedExtractValue = extractValue
+    return Column(expr)
+  }
+
   private func sortOrder(
     _ direction: Spark_Connect_Expression.SortOrder.SortDirection,
     _ nullOrdering: Spark_Connect_Expression.SortOrder.NullOrdering
@@ -100,6 +322,10 @@ public struct Column: Sendable {
   }
 
   private static func fn(_ name: String, _ args: Column...) -> Column {
+    return fn(name, args)
+  }
+
+  private static func fn(_ name: String, _ args: [Column]) -> Column {
     var function = Spark_Connect_Expression.UnresolvedFunction()
     function.functionName = name
     function.arguments = args.map { $0.expr }
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift 
b/Tests/SparkConnectTests/FunctionsTests.swift
index e3ff0fb..3d18289 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -230,6 +230,84 @@ struct FunctionsTests {
     
#expect(expr.unresolvedFunction.arguments[1].unresolvedFunction.functionName == 
"!")
   }
 
+  @Test
+  func predicates() throws {
+    let isNull = col("a").isNull().expr
+    #expect(isNull.unresolvedFunction.functionName == "isnull")
+    #expect(isNull.unresolvedFunction.arguments.count == 1)
+    
#expect(isNull.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+
+    let isNotNull = col("a").isNotNull().expr
+    #expect(isNotNull.unresolvedFunction.functionName == "isnotnull")
+    #expect(isNotNull.unresolvedFunction.arguments.count == 1)
+
+    let isin = col("a").isin(1, 2, "x").expr
+    #expect(isin.unresolvedFunction.functionName == "in")
+    #expect(isin.unresolvedFunction.arguments.count == 4)
+    
#expect(isin.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+    #expect(isin.unresolvedFunction.arguments[1].literal.long == 1)
+    #expect(isin.unresolvedFunction.arguments[3].literal.string == "x")
+
+    let eqNullSafe = col("a").eqNullSafe(col("b")).expr
+    #expect(eqNullSafe.unresolvedFunction.functionName == "<=>")
+    #expect(eqNullSafe.unresolvedFunction.arguments.count == 2)
+    
#expect(col("a").eqNullSafe("x").expr.unresolvedFunction.arguments[1].literal.string
 == "x")
+  }
+
+  @Test
+  func between() throws {
+    // Composed as (a >= 1) and (a <= 10)
+    let expr = col("a").between(1, 10).expr
+    #expect(expr.unresolvedFunction.functionName == "and")
+    let lower = expr.unresolvedFunction.arguments[0].unresolvedFunction
+    #expect(lower.functionName == ">=")
+    #expect(lower.arguments[1].literal.long == 1)
+    let upper = expr.unresolvedFunction.arguments[1].unresolvedFunction
+    #expect(upper.functionName == "<=")
+    #expect(upper.arguments[1].literal.long == 10)
+
+    #expect(col("a").between(col("b"), 
col("c")).expr.unresolvedFunction.functionName == "and")
+    #expect(col("a").between(col("b"), 
10).expr.unresolvedFunction.functionName == "and")
+    #expect(col("a").between(1, col("c")).expr.unresolvedFunction.functionName 
== "and")
+  }
+
+  @Test
+  func stringMethods() throws {
+    for (column, name) in [
+      (col("a").like("x%"), "like"),
+      (col("a").rlike("^x"), "rlike"),
+      (col("a").ilike("x%"), "ilike"),
+      (col("a").contains(col("b")), "contains"),
+      (col("a").contains("x"), "contains"),
+      (col("a").startsWith(col("b")), "startswith"),
+      (col("a").startsWith("x"), "startswith"),
+      (col("a").endsWith(col("b")), "endswith"),
+      (col("a").endsWith("x"), "endswith"),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments.count == 2)
+      
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
 == "a")
+    }
+
+    let substr = col("a").substr(1, 3).expr
+    #expect(substr.unresolvedFunction.functionName == "substr")
+    #expect(substr.unresolvedFunction.arguments.count == 3)
+    #expect(substr.unresolvedFunction.arguments[1].literal.long == 1)
+    #expect(substr.unresolvedFunction.arguments[2].literal.long == 3)
+  }
+
+  @Test
+  func extraction() throws {
+    let item = col("a").getItem(0).expr
+    
#expect(item.unresolvedExtractValue.child.unresolvedAttribute.unparsedIdentifier
 == "a")
+    #expect(item.unresolvedExtractValue.extraction.literal.long == 0)
+
+    let field = col("a").getField("b").expr
+    
#expect(field.unresolvedExtractValue.child.unresolvedAttribute.unparsedIdentifier
 == "a")
+    #expect(field.unresolvedExtractValue.extraction.literal.string == "b")
+  }
+
   @Test
   func selectColumns() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -291,4 +369,47 @@ struct FunctionsTests {
     #expect(rows == [Row(2, 0, 2, -1), Row(3, 1, 4, -2), Row(4, 2, 6, -3)])
     await spark.stop()
   }
+
+  @Test
+  func filterWithPredicates() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql(
+      "SELECT * FROM VALUES ('Alice', 20), ('Bob', NULL), (NULL, 30) T(name, 
age)")
+    #expect(try await df.filter(col("name").isNull()).count() == 1)
+    #expect(try await df.filter(col("name").isNotNull()).count() == 2)
+    #expect(try await df.filter(col("age").isin(20, 30)).count() == 2)
+    #expect(try await df.filter(col("age").between(20, 30)).count() == 2)
+    #expect(try await df.filter(col("name").isNotNull() && 
col("age").between(20, 30)).count() == 1)
+
+    let pairs = try await spark.sql("SELECT * FROM VALUES ('a', 'a'), (NULL, 
NULL), ('a', NULL) T(x, y)")
+    #expect(try await pairs.filter(col("x").eqNullSafe(col("y"))).count() == 2)
+    #expect(try await pairs.filter(col("x").eqNullSafe("a")).count() == 2)
+    await spark.stop()
+  }
+
+  @Test
+  func filterWithStringMethods() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql("SELECT * FROM VALUES ('Alice'), ('Bob'), 
('Charlie') T(name)")
+    #expect(try await df.filter(col("name").like("Al%")).count() == 1)
+    #expect(try await df.filter(col("name").rlike("^.o.$")).count() == 1)
+    #expect(try await df.filter(col("name").ilike("alice")).count() == 1)
+    #expect(try await df.filter(col("name").contains("li")).count() == 2)
+    #expect(try await df.filter(col("name").startsWith("B")).count() == 1)
+    #expect(try await df.filter(col("name").endsWith("e")).count() == 2)
+    await spark.stop()
+  }
+
+  @Test
+  func selectWithSubstrAndExtraction() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql(
+      "SELECT 'Alice' AS name, array(1, 2, 3) AS arr, map('k', 10) AS m, 
named_struct('a', 7) AS s")
+    #expect(try await df.select(col("name").substr(1, 3)).collect() == 
[Row("Ali")])
+    #expect(try await df.select(col("name").substr(lit(2), lit(3))).collect() 
== [Row("lic")])
+    let rows = try await df.select(
+      col("arr").getItem(0), col("m").getItem("k"), 
col("s").getField("a")).collect()
+    #expect(rows == [Row(1, 10, 7)])
+    await spark.stop()
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to