This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 559fe2a [SPARK-58474] Support Swift literal operands in `Column`
operators via `SparkLiteral`
559fe2a is described below
commit 559fe2adac536e4f8430272c22f4b3afb30c002f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 08:52:59 2026 -0700
[SPARK-58474] Support Swift literal operands in `Column` operators via
`SparkLiteral`
### What changes were proposed in this pull request?
This PR aims to support Swift literal operands in `Column` operators so
that literals can be mixed with `Column` expressions directly without wrapping
them in `lit`.
- A new public protocol `SparkLiteral` with a `toLiteralColumn`
requirement, conformed by all nine types supported by `lit`: `Bool`, `Int8`,
`Int16`, `Int32`, `Int64`, `Int`, `Float`, `Double`, and `String`.
- Generic overloads for all thirteen binary operators (`==`, `!=`, `<`,
`<=`, `>`, `>=`, `&&`, `||`, `+`, `-`, `*`, `/`, `%`) accepting `some
SparkLiteral` on either the left-hand or right-hand side.
```swift
df.filter(col("age") > 21 && col("name") == "Alice")
df.select((col("id") * 2).alias("doubled"))
df.filter(21 < col("age"))
```
Each conformance reuses the existing `lit` functions, so the
literal-to-proto mapping is defined in one place. Existing Swift operations
such as `Int == Int` are not affected because every overload requires a
`Column` on one side.
### Why are the changes needed?
This allows users to write expressions in the idiomatic PySpark/Scala style
(`col("age") > 21`) instead of `col("age") > lit(21)`, without an overload
explosion per literal type.
### Does this PR introduce _any_ user-facing change?
No. This is a new feature addition to the unreleased `Column` API.
### How was this patch tested?
Pass the CIs with the newly added unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #456 from dongjoon-hyun/SPARK-58474.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Column.swift | 136 +++++++++++++++++++++++++++
Sources/SparkConnect/Functions.swift | 49 ++++++++++
Tests/SparkConnectTests/FunctionsTests.swift | 58 ++++++++++++
3 files changed, 243 insertions(+)
diff --git a/Sources/SparkConnect/Column.swift
b/Sources/SparkConnect/Column.swift
index 95be1d2..b842663 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -197,4 +197,140 @@ public struct Column: Sendable {
public static prefix func - (col: Column) -> Column {
return fn("negative", col)
}
+
+ // MARK: - Literal operand overloads
+
+ /// Returns an equality test expression against a literal value.
+ ///
+ /// ```swift
+ /// df.filter(col("name") == "Alice")
+ /// ```
+ public static func == (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs == rhs.toLiteralColumn
+ }
+
+ /// Returns an equality test expression against a literal value.
+ public static func == (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn == rhs
+ }
+
+ /// Returns an inequality test expression against a literal value.
+ public static func != (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs != rhs.toLiteralColumn
+ }
+
+ /// Returns an inequality test expression against a literal value.
+ public static func != (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn != rhs
+ }
+
+ /// Returns a less-than test expression against a literal value.
+ public static func < (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs < rhs.toLiteralColumn
+ }
+
+ /// Returns a less-than test expression against a literal value.
+ public static func < (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn < rhs
+ }
+
+ /// Returns a less-than-or-equal test expression against a literal value.
+ public static func <= (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs <= rhs.toLiteralColumn
+ }
+
+ /// Returns a less-than-or-equal test expression against a literal value.
+ public static func <= (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn <= rhs
+ }
+
+ /// Returns a greater-than test expression against a literal value.
+ public static func > (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs > rhs.toLiteralColumn
+ }
+
+ /// Returns a greater-than test expression against a literal value.
+ public static func > (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn > rhs
+ }
+
+ /// Returns a greater-than-or-equal test expression against a literal value.
+ public static func >= (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs >= rhs.toLiteralColumn
+ }
+
+ /// Returns a greater-than-or-equal test expression against a literal value.
+ public static func >= (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn >= rhs
+ }
+
+ /// Returns a logical AND expression with a literal value.
+ public static func && (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs && rhs.toLiteralColumn
+ }
+
+ /// Returns a logical AND expression with a literal value.
+ public static func && (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn && rhs
+ }
+
+ /// Returns a logical OR expression with a literal value.
+ public static func || (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs || rhs.toLiteralColumn
+ }
+
+ /// Returns a logical OR expression with a literal value.
+ public static func || (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn || rhs
+ }
+
+ /// Returns an addition expression with a literal value.
+ public static func + (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs + rhs.toLiteralColumn
+ }
+
+ /// Returns an addition expression with a literal value.
+ public static func + (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn + rhs
+ }
+
+ /// Returns a subtraction expression with a literal value.
+ public static func - (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs - rhs.toLiteralColumn
+ }
+
+ /// Returns a subtraction expression with a literal value.
+ public static func - (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn - rhs
+ }
+
+ /// Returns a multiplication expression with a literal value.
+ public static func * (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs * rhs.toLiteralColumn
+ }
+
+ /// Returns a multiplication expression with a literal value.
+ public static func * (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn * rhs
+ }
+
+ /// Returns a division expression with a literal value.
+ public static func / (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs / rhs.toLiteralColumn
+ }
+
+ /// Returns a division expression with a literal value.
+ public static func / (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn / rhs
+ }
+
+ /// Returns a modulo expression with a literal value.
+ public static func % (lhs: Column, rhs: some SparkLiteral) -> Column {
+ return lhs % rhs.toLiteralColumn
+ }
+
+ /// Returns a modulo expression with a literal value.
+ public static func % (lhs: some SparkLiteral, rhs: Column) -> Column {
+ return lhs.toLiteralColumn % rhs
+ }
}
diff --git a/Sources/SparkConnect/Functions.swift
b/Sources/SparkConnect/Functions.swift
index b05fd5a..b3c97fb 100644
--- a/Sources/SparkConnect/Functions.swift
+++ b/Sources/SparkConnect/Functions.swift
@@ -112,6 +112,55 @@ public func lit(_ value: String) -> Column {
return litColumn(literal)
}
+/// A type that can be used as a literal operand of ``Column`` operators.
+///
+/// This allows Swift literals to be mixed with ``Column`` expressions directly
+/// without wrapping them in `lit`:
+///
+/// ```swift
+/// df.filter(col("age") > 21 && col("name") == "Alice")
+/// ```
+public protocol SparkLiteral {
+ /// A literal ``Column`` representation of this value.
+ var toLiteralColumn: Column { get }
+}
+
+extension Bool: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Int8: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Int16: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Int32: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Int64: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Int: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Float: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension Double: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
+extension String: SparkLiteral {
+ public var toLiteralColumn: Column { lit(self) }
+}
+
/// Returns a sort expression based on the ascending order of the given column
name.
/// - Parameter name: A column name.
/// - Returns: A ``Column`` with ascending sort order.
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift
b/Tests/SparkConnectTests/FunctionsTests.swift
index 16cee8c..e3ff0fb 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -172,6 +172,52 @@ struct FunctionsTests {
#expect(negate.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
}
+ @Test
+ func literalOperands() throws {
+ for (column, name) in [
+ (col("a") == "x", "="),
+ (col("a") < 1, "<"),
+ (col("a") <= 1, "<="),
+ (col("a") > 1, ">"),
+ (col("a") >= 1, ">="),
+ (col("a") && true, "and"),
+ (col("a") || true, "or"),
+ (col("a") + 1, "+"),
+ (col("a") - 1, "-"),
+ (col("a") * 1, "*"),
+ (col("a") / 1, "/"),
+ (col("a") % 1, "%"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ if case .literal = expr.unresolvedFunction.arguments[1].exprType {
+ } else {
+ Issue.record("Expected a literal argument: \(name)")
+ }
+ }
+
+ // The literal can also be used on the left-hand side.
+ let lhs = (21 < col("age")).expr
+ #expect(lhs.unresolvedFunction.functionName == "<")
+ #expect(lhs.unresolvedFunction.arguments[0].literal.long == 21)
+
#expect(lhs.unresolvedFunction.arguments[1].unresolvedAttribute.unparsedIdentifier
== "age")
+ }
+
+ @Test
+ func literalOperandTypes() throws {
+ #expect((col("a") ==
true).expr.unresolvedFunction.arguments[1].literal.boolean == true)
+ #expect((col("a") ==
Int8(1)).expr.unresolvedFunction.arguments[1].literal.byte == 1)
+ #expect((col("a") ==
Int16(1)).expr.unresolvedFunction.arguments[1].literal.short == 1)
+ #expect((col("a") ==
Int32(1)).expr.unresolvedFunction.arguments[1].literal.integer == 1)
+ #expect((col("a") ==
Int64(1)).expr.unresolvedFunction.arguments[1].literal.long == 1)
+ #expect((col("a") == 1).expr.unresolvedFunction.arguments[1].literal.long
== 1)
+ #expect((col("a") ==
Float(1.0)).expr.unresolvedFunction.arguments[1].literal.float == 1.0)
+ #expect((col("a") ==
1.0).expr.unresolvedFunction.arguments[1].literal.double == 1.0)
+ #expect((col("a") ==
"x").expr.unresolvedFunction.arguments[1].literal.string == "x")
+ }
+
@Test
func operatorPrecedence() throws {
// Parsed as ((a > b) and (c = d)) or (not e)
@@ -220,6 +266,18 @@ struct FunctionsTests {
await spark.stop()
}
+ @Test
+ func filterWithLiteralOperands() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(10)
+ #expect(try await df.filter(col("id") == 3).collect() == [Row(3)])
+ #expect(try await df.filter(col("id") > 2 && col("id") < 5).count() == 2)
+ #expect(try await df.filter(2 > col("id")).count() == 2)
+ #expect(try await df.where(col("id") % 3 == 0).count() == 4)
+ #expect(try await df.select((col("id") +
1).alias("plus")).filter(col("plus") == 10).count() == 1)
+ await spark.stop()
+ }
+
@Test
func selectWithOperators() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]