This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new c7ace77 [SPARK-58475] Support predicate and string methods in `Column`
c7ace77 is described below
commit c7ace776868bd19fe05b3e5ed037d49a1aab5e0f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 10:36:07 2026 -0700
[SPARK-58475] Support predicate and string methods in `Column`
### What changes were proposed in this pull request?
This PR aims to support predicate, string, and extraction methods in
`Column`.
- Predicates: `isNull`, `isNotNull`, `isin`, `between`, `eqNullSafe`
- String methods: `like`, `rlike`, `ilike`, `contains`, `startsWith`,
`endsWith`, `substr`
- Extraction: `getItem`, `getField`
The function name mappings follow the Scala client's `Column.scala`:
`isnull`/`isnotnull`, `in`, `<=>`, `like`/`rlike`/`ilike`,
`contains`/`startswith`/`endswith`, `substr`. `between` is composed as `(self
>= lower) && (self <= upper)` like the Scala client, and `getItem`/`getField`
use `Expression.UnresolvedExtractValue` like the Scala client's
`apply(extraction)`. Methods taking operands accept both `Column` and
`SparkLiteral` values.
### Why are the changes needed?
To allow Swift users to build filter and projection expressions
idiomatically without falling back to SQL strings.
```swift
df.filter(col("name").isNotNull() && col("age").between(20, 30))
df.filter(col("name").like("Al%"))
df.select(col("arr").getItem(0), col("s").getField("a"))
```
### Does this PR introduce _any_ user-facing change?
No. This is a new API addition.
### How was this patch tested?
Pass the CIs with newly added unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #457 from dongjoon-hyun/SPARK-58475.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Column.swift | 226 +++++++++++++++++++++++++++
Tests/SparkConnectTests/FunctionsTests.swift | 121 ++++++++++++++
2 files changed, 347 insertions(+)
diff --git a/Sources/SparkConnect/Column.swift
b/Sources/SparkConnect/Column.swift
index b842663..d7663f0 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -86,6 +86,228 @@ public struct Column: Sendable {
return Column(expr)
}
+ // MARK: - Predicates
+
+ /// Returns an expression that is true if this column is null.
+ ///
+ /// ```swift
+ /// df.filter(col("name").isNull())
+ /// ```
+ /// - Returns: A ``Column`` testing for null.
+ public func isNull() -> Column {
+ return Column.fn("isnull", self)
+ }
+
+ /// Returns an expression that is true if this column is not null.
+ ///
+ /// ```swift
+ /// df.filter(col("name").isNotNull())
+ /// ```
+ /// - Returns: A ``Column`` testing for non-null.
+ public func isNotNull() -> Column {
+ return Column.fn("isnotnull", self)
+ }
+
+ /// Returns an expression that is true if this column's value is contained in
+ /// the given values.
+ ///
+ /// ```swift
+ /// df.filter(col("age").isin(20, 30))
+ /// ```
+ /// - Parameter values: Literal values to test against.
+ /// - Returns: A ``Column`` testing for membership.
+ public func isin(_ values: any SparkLiteral...) -> Column {
+ return Column.fn("in", [self] + values.map { $0.toLiteralColumn })
+ }
+
+ /// Returns an expression that is true if this column is between the given
+ /// lower and upper bounds, inclusive.
+ ///
+ /// ```swift
+ /// df.filter(col("age").between(20, 30))
+ /// ```
+ /// - Parameters:
+ /// - lower: A lower bound.
+ /// - upper: An upper bound.
+ /// - Returns: A ``Column`` testing for the range.
+ public func between(_ lower: Column, _ upper: Column) -> Column {
+ return (self >= lower) && (self <= upper)
+ }
+
+ /// Returns an expression that is true if this column is between the given
+ /// lower and upper bounds, inclusive.
+ public func between(_ lower: Column, _ upper: some SparkLiteral) -> Column {
+ return (self >= lower) && (self <= upper)
+ }
+
+ /// Returns an expression that is true if this column is between the given
+ /// lower and upper bounds, inclusive.
+ public func between(_ lower: some SparkLiteral, _ upper: Column) -> Column {
+ return (self >= lower) && (self <= upper)
+ }
+
+ /// Returns an expression that is true if this column is between the given
+ /// lower and upper bounds, inclusive.
+ public func between(_ lower: some SparkLiteral, _ upper: some SparkLiteral)
-> Column {
+ return (self >= lower) && (self <= upper)
+ }
+
+ /// Returns a null-safe equality test expression. Unlike `==`, this returns
+ /// `true` when both sides are null.
+ ///
+ /// ```swift
+ /// df.filter(col("name").eqNullSafe(col("nickname")))
+ /// ```
+ /// - Parameter other: A ``Column`` to compare with.
+ /// - Returns: A ``Column`` testing for null-safe equality.
+ public func eqNullSafe(_ other: Column) -> Column {
+ return Column.fn("<=>", self, other)
+ }
+
+ /// Returns a null-safe equality test expression against a literal value.
+ public func eqNullSafe(_ other: some SparkLiteral) -> Column {
+ return eqNullSafe(other.toLiteralColumn)
+ }
+
+ // MARK: - String methods
+
+ /// Returns a SQL `LIKE` expression.
+ ///
+ /// ```swift
+ /// df.filter(col("name").like("Al%"))
+ /// ```
+ /// - Parameter pattern: A SQL LIKE pattern.
+ /// - Returns: A ``Column`` testing for the pattern match.
+ public func like(_ pattern: String) -> Column {
+ return Column.fn("like", self, pattern.toLiteralColumn)
+ }
+
+ /// Returns a SQL `RLIKE` expression (LIKE with regex).
+ ///
+ /// ```swift
+ /// df.filter(col("name").rlike("^Al.*"))
+ /// ```
+ /// - Parameter pattern: A regular expression.
+ /// - Returns: A ``Column`` testing for the regex match.
+ public func rlike(_ pattern: String) -> Column {
+ return Column.fn("rlike", self, pattern.toLiteralColumn)
+ }
+
+ /// Returns a SQL `ILIKE` expression (case insensitive LIKE).
+ ///
+ /// ```swift
+ /// df.filter(col("name").ilike("al%"))
+ /// ```
+ /// - Parameter pattern: A SQL LIKE pattern.
+ /// - Returns: A ``Column`` testing for the case-insensitive pattern match.
+ public func ilike(_ pattern: String) -> Column {
+ return Column.fn("ilike", self, pattern.toLiteralColumn)
+ }
+
+ /// Returns an expression that is true if this column contains the other
value.
+ ///
+ /// ```swift
+ /// df.filter(col("name").contains("li"))
+ /// ```
+ /// - Parameter other: A ``Column`` to search for.
+ /// - Returns: A ``Column`` testing for containment.
+ public func contains(_ other: Column) -> Column {
+ return Column.fn("contains", self, other)
+ }
+
+ /// Returns an expression that is true if this column contains the literal
value.
+ public func contains(_ other: some SparkLiteral) -> Column {
+ return contains(other.toLiteralColumn)
+ }
+
+ /// Returns an expression that is true if this column starts with the other
value.
+ ///
+ /// ```swift
+ /// df.filter(col("name").startsWith("Al"))
+ /// ```
+ /// - Parameter other: A ``Column`` prefix.
+ /// - Returns: A ``Column`` testing for the prefix match.
+ public func startsWith(_ other: Column) -> Column {
+ return Column.fn("startswith", self, other)
+ }
+
+ /// Returns an expression that is true if this column starts with the
literal string.
+ public func startsWith(_ literal: String) -> Column {
+ return startsWith(literal.toLiteralColumn)
+ }
+
+ /// Returns an expression that is true if this column ends with the other
value.
+ ///
+ /// ```swift
+ /// df.filter(col("name").endsWith("ce"))
+ /// ```
+ /// - Parameter other: A ``Column`` suffix.
+ /// - Returns: A ``Column`` testing for the suffix match.
+ public func endsWith(_ other: Column) -> Column {
+ return Column.fn("endswith", self, other)
+ }
+
+ /// Returns an expression that is true if this column ends with the literal
string.
+ public func endsWith(_ literal: String) -> Column {
+ return endsWith(literal.toLiteralColumn)
+ }
+
+ /// Returns a substring expression.
+ ///
+ /// ```swift
+ /// df.select(col("name").substr(lit(1), lit(3)))
+ /// ```
+ /// - Parameters:
+ /// - startPos: A starting position (1-based).
+ /// - len: A substring length.
+ /// - Returns: A ``Column`` of the substring.
+ public func substr(_ startPos: Column, _ len: Column) -> Column {
+ return Column.fn("substr", self, startPos, len)
+ }
+
+ /// Returns a substring expression.
+ ///
+ /// ```swift
+ /// df.select(col("name").substr(1, 3))
+ /// ```
+ public func substr(_ startPos: Int, _ len: Int) -> Column {
+ return substr(startPos.toLiteralColumn, len.toLiteralColumn)
+ }
+
+ // MARK: - Extraction
+
+ /// Returns an expression that gets an item at the given position out of an
+ /// array, or gets a value by the given key out of a map.
+ ///
+ /// ```swift
+ /// df.select(col("arr").getItem(0), col("map").getItem("key"))
+ /// ```
+ /// - Parameter key: An array position (0-based) or a map key.
+ /// - Returns: A ``Column`` of the extracted value.
+ public func getItem(_ key: some SparkLiteral) -> Column {
+ return extractValue(key.toLiteralColumn)
+ }
+
+ /// Returns an expression that gets a field by the given name in a struct.
+ ///
+ /// ```swift
+ /// df.select(col("struct").getField("a"))
+ /// ```
+ /// - Parameter name: A field name.
+ /// - Returns: A ``Column`` of the extracted field.
+ public func getField(_ name: String) -> Column {
+ return extractValue(name.toLiteralColumn)
+ }
+
+ private func extractValue(_ extraction: Column) -> Column {
+ var extractValue = Spark_Connect_Expression.UnresolvedExtractValue()
+ extractValue.child = self.expr
+ extractValue.extraction = extraction.expr
+ var expr = Spark_Connect_Expression()
+ expr.unresolvedExtractValue = extractValue
+ return Column(expr)
+ }
+
private func sortOrder(
_ direction: Spark_Connect_Expression.SortOrder.SortDirection,
_ nullOrdering: Spark_Connect_Expression.SortOrder.NullOrdering
@@ -100,6 +322,10 @@ public struct Column: Sendable {
}
private static func fn(_ name: String, _ args: Column...) -> Column {
+ return fn(name, args)
+ }
+
+ private static func fn(_ name: String, _ args: [Column]) -> Column {
var function = Spark_Connect_Expression.UnresolvedFunction()
function.functionName = name
function.arguments = args.map { $0.expr }
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift
b/Tests/SparkConnectTests/FunctionsTests.swift
index e3ff0fb..3d18289 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -230,6 +230,84 @@ struct FunctionsTests {
#expect(expr.unresolvedFunction.arguments[1].unresolvedFunction.functionName ==
"!")
}
+ @Test
+ func predicates() throws {
+ let isNull = col("a").isNull().expr
+ #expect(isNull.unresolvedFunction.functionName == "isnull")
+ #expect(isNull.unresolvedFunction.arguments.count == 1)
+
#expect(isNull.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+
+ let isNotNull = col("a").isNotNull().expr
+ #expect(isNotNull.unresolvedFunction.functionName == "isnotnull")
+ #expect(isNotNull.unresolvedFunction.arguments.count == 1)
+
+ let isin = col("a").isin(1, 2, "x").expr
+ #expect(isin.unresolvedFunction.functionName == "in")
+ #expect(isin.unresolvedFunction.arguments.count == 4)
+
#expect(isin.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ #expect(isin.unresolvedFunction.arguments[1].literal.long == 1)
+ #expect(isin.unresolvedFunction.arguments[3].literal.string == "x")
+
+ let eqNullSafe = col("a").eqNullSafe(col("b")).expr
+ #expect(eqNullSafe.unresolvedFunction.functionName == "<=>")
+ #expect(eqNullSafe.unresolvedFunction.arguments.count == 2)
+
#expect(col("a").eqNullSafe("x").expr.unresolvedFunction.arguments[1].literal.string
== "x")
+ }
+
+ @Test
+ func between() throws {
+ // Composed as (a >= 1) and (a <= 10)
+ let expr = col("a").between(1, 10).expr
+ #expect(expr.unresolvedFunction.functionName == "and")
+ let lower = expr.unresolvedFunction.arguments[0].unresolvedFunction
+ #expect(lower.functionName == ">=")
+ #expect(lower.arguments[1].literal.long == 1)
+ let upper = expr.unresolvedFunction.arguments[1].unresolvedFunction
+ #expect(upper.functionName == "<=")
+ #expect(upper.arguments[1].literal.long == 10)
+
+ #expect(col("a").between(col("b"),
col("c")).expr.unresolvedFunction.functionName == "and")
+ #expect(col("a").between(col("b"),
10).expr.unresolvedFunction.functionName == "and")
+ #expect(col("a").between(1, col("c")).expr.unresolvedFunction.functionName
== "and")
+ }
+
+ @Test
+ func stringMethods() throws {
+ for (column, name) in [
+ (col("a").like("x%"), "like"),
+ (col("a").rlike("^x"), "rlike"),
+ (col("a").ilike("x%"), "ilike"),
+ (col("a").contains(col("b")), "contains"),
+ (col("a").contains("x"), "contains"),
+ (col("a").startsWith(col("b")), "startswith"),
+ (col("a").startsWith("x"), "startswith"),
+ (col("a").endsWith(col("b")), "endswith"),
+ (col("a").endsWith("x"), "endswith"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ }
+
+ let substr = col("a").substr(1, 3).expr
+ #expect(substr.unresolvedFunction.functionName == "substr")
+ #expect(substr.unresolvedFunction.arguments.count == 3)
+ #expect(substr.unresolvedFunction.arguments[1].literal.long == 1)
+ #expect(substr.unresolvedFunction.arguments[2].literal.long == 3)
+ }
+
+ @Test
+ func extraction() throws {
+ let item = col("a").getItem(0).expr
+
#expect(item.unresolvedExtractValue.child.unresolvedAttribute.unparsedIdentifier
== "a")
+ #expect(item.unresolvedExtractValue.extraction.literal.long == 0)
+
+ let field = col("a").getField("b").expr
+
#expect(field.unresolvedExtractValue.child.unresolvedAttribute.unparsedIdentifier
== "a")
+ #expect(field.unresolvedExtractValue.extraction.literal.string == "b")
+ }
+
@Test
func selectColumns() async throws {
let spark = try await SparkSession.builder.getOrCreate()
@@ -291,4 +369,47 @@ struct FunctionsTests {
#expect(rows == [Row(2, 0, 2, -1), Row(3, 1, 4, -2), Row(4, 2, 6, -3)])
await spark.stop()
}
+
+ @Test
+ func filterWithPredicates() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES ('Alice', 20), ('Bob', NULL), (NULL, 30) T(name,
age)")
+ #expect(try await df.filter(col("name").isNull()).count() == 1)
+ #expect(try await df.filter(col("name").isNotNull()).count() == 2)
+ #expect(try await df.filter(col("age").isin(20, 30)).count() == 2)
+ #expect(try await df.filter(col("age").between(20, 30)).count() == 2)
+ #expect(try await df.filter(col("name").isNotNull() &&
col("age").between(20, 30)).count() == 1)
+
+ let pairs = try await spark.sql("SELECT * FROM VALUES ('a', 'a'), (NULL,
NULL), ('a', NULL) T(x, y)")
+ #expect(try await pairs.filter(col("x").eqNullSafe(col("y"))).count() == 2)
+ #expect(try await pairs.filter(col("x").eqNullSafe("a")).count() == 2)
+ await spark.stop()
+ }
+
+ @Test
+ func filterWithStringMethods() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql("SELECT * FROM VALUES ('Alice'), ('Bob'),
('Charlie') T(name)")
+ #expect(try await df.filter(col("name").like("Al%")).count() == 1)
+ #expect(try await df.filter(col("name").rlike("^.o.$")).count() == 1)
+ #expect(try await df.filter(col("name").ilike("alice")).count() == 1)
+ #expect(try await df.filter(col("name").contains("li")).count() == 2)
+ #expect(try await df.filter(col("name").startsWith("B")).count() == 1)
+ #expect(try await df.filter(col("name").endsWith("e")).count() == 2)
+ await spark.stop()
+ }
+
+ @Test
+ func selectWithSubstrAndExtraction() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT 'Alice' AS name, array(1, 2, 3) AS arr, map('k', 10) AS m,
named_struct('a', 7) AS s")
+ #expect(try await df.select(col("name").substr(1, 3)).collect() ==
[Row("Ali")])
+ #expect(try await df.select(col("name").substr(lit(2), lit(3))).collect()
== [Row("lic")])
+ let rows = try await df.select(
+ col("arr").getItem(0), col("m").getItem("k"),
col("s").getField("a")).collect()
+ #expect(rows == [Row(1, 10, 7)])
+ await spark.stop()
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]