This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 81d155e [SPARK-58421] Improve `sql` parameter binding to support more
literal types instead of crashing
81d155e is described below
commit 81d155ee5ad1de1ea5cd4c32205a27077f34475a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 10:55:36 2026 -0700
[SPARK-58421] Improve `sql` parameter binding to support more literal types
instead of crashing
### What changes were proposed in this pull request?
This PR improves the argument conversion of `spark.sql(_:args:)` parameter
binding.
1. Replace the crashing `value as! String` fallback with throwing
`SparkConnectError.InvalidType`.
2. Support more types: `Float`, `Double`, `Decimal`, `Data` (binary),
`Date` (timestamp), and `nil` (null).
### Why are the changes needed?
Previously, unsupported argument types (including `Float` and `Double`)
crashed the process
via a force cast instead of returning a catchable error.
### Does this PR introduce _any_ user-facing change?
No. Unsupported argument types now throw `SparkConnectError.InvalidType`
instead of crashing.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #447 from dongjoon-hyun/SPARK-58421.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/DataFrame.swift | 4 +--
Sources/SparkConnect/Extension.swift | 32 +++++++++++++++++-----
Tests/SparkConnectTests/SparkSessionTests.swift | 36 +++++++++++++++++++++++++
3 files changed, 64 insertions(+), 8 deletions(-)
diff --git a/Sources/SparkConnect/DataFrame.swift
b/Sources/SparkConnect/DataFrame.swift
index 15fdad6..98b6fda 100644
--- a/Sources/SparkConnect/DataFrame.swift
+++ b/Sources/SparkConnect/DataFrame.swift
@@ -222,7 +222,7 @@ public actor DataFrame: Sendable {
init(spark: SparkSession, sqlText: String, _ posArgs: [Sendable]? = nil)
async throws {
self.spark = spark
if let posArgs {
- self.plan = sqlText.toSparkConnectPlan(posArgs)
+ self.plan = try sqlText.toSparkConnectPlan(posArgs)
} else {
self.plan = sqlText.toSparkConnectPlan
}
@@ -230,7 +230,7 @@ public actor DataFrame: Sendable {
init(spark: SparkSession, sqlText: String, _ args: [String: Sendable]) async
throws {
self.spark = spark
- self.plan = sqlText.toSparkConnectPlan(args)
+ self.plan = try sqlText.toSparkConnectPlan(args)
}
deinit {
diff --git a/Sources/SparkConnect/Extension.swift
b/Sources/SparkConnect/Extension.swift
index cc91503..32da34c 100644
--- a/Sources/SparkConnect/Extension.swift
+++ b/Sources/SparkConnect/Extension.swift
@@ -35,7 +35,7 @@ extension String {
return plan
}
- private func toExpression(_ value: Sendable) -> Spark_Connect_Expression {
+ private func toExpression(_ value: Sendable) throws ->
Spark_Connect_Expression {
var literal = ExpressionLiteral()
switch value {
case let value as Bool:
@@ -50,20 +50,40 @@ extension String {
literal.long = value
case let value as Int:
literal.long = Int64(value)
+ case let value as Float:
+ literal.float = value
+ case let value as Double:
+ literal.double = value
+ case let value as Decimal:
+ var decimal = ExpressionLiteral.Decimal()
+ decimal.value = "\(value)"
+ literal.decimal = decimal
case let value as String:
literal.string = value
+ case let value as Data:
+ literal.binary = value
+ case let value as Date:
+ // `Date` is an absolute point in time, so it maps to Spark's `TIMESTAMP`
+ // (microseconds since the UNIX epoch), not `DATE`.
+ literal.timestamp = Int64(value.timeIntervalSince1970 * 1_000_000)
default:
- literal.string = value as! String
+ if case Optional<Any>.none = value as Any {
+ var dataType = DataType()
+ dataType.null = DataType.NULL()
+ literal.null = dataType
+ } else {
+ throw SparkConnectError.InvalidType
+ }
}
var expr = Spark_Connect_Expression()
expr.literal = literal
return expr
}
- func toSparkConnectPlan(_ posArguments: [Sendable]) -> Plan {
+ func toSparkConnectPlan(_ posArguments: [Sendable]) throws -> Plan {
var sql = Spark_Connect_SQL()
sql.query = self
- sql.posArguments = posArguments.map { toExpression($0) }
+ sql.posArguments = try posArguments.map { try toExpression($0) }
var relation = Relation()
relation.sql = sql
var plan = Plan()
@@ -71,10 +91,10 @@ extension String {
return plan
}
- func toSparkConnectPlan(_ namedArguments: [String: Sendable]) -> Plan {
+ func toSparkConnectPlan(_ namedArguments: [String: Sendable]) throws -> Plan
{
var sql = Spark_Connect_SQL()
sql.query = self
- sql.namedArguments = namedArguments.mapValues { toExpression($0) }
+ sql.namedArguments = try namedArguments.mapValues { try toExpression($0) }
var relation = Relation()
relation.sql = sql
var plan = Plan()
diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift
b/Tests/SparkConnectTests/SparkSessionTests.swift
index ae2eba9..64ab712 100644
--- a/Tests/SparkConnectTests/SparkSessionTests.swift
+++ b/Tests/SparkConnectTests/SparkSessionTests.swift
@@ -148,6 +148,42 @@ struct SparkSessionTests {
await spark.stop()
}
+ @Test
+ func sqlWithMoreParameterTypes() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ if await spark.version.starts(with: "4.") {
+ // Use at most 4 positional parameters per query because Spark Connect
+ // servers (up to 4.2.0) bind 5 or more positional parameters in the
wrong order.
+ #expect(
+ try await spark.sql(
+ "SELECT typeof(?), typeof(?), typeof(?)", Float(1.0), 2.0,
Decimal(string: "3.1")!
+ ).collect() == [Row("float", "double", "decimal(2,1)")])
+ #expect(
+ try await spark.sql(
+ "SELECT typeof(?), typeof(?), typeof(?)", Data([1, 2, 3]), Date(),
nil as String?
+ ).collect() == [Row("binary", "timestamp", "void")])
+ #expect(try await spark.sql("SELECT ? * ?", Float(2.0), 3.0).collect()
== [Row(6.0)])
+ #expect(
+ try await spark.sql("SELECT CAST(:x AS STRING)", args: ["x": nil as
String?]).collect()
+ == [Row(nil)])
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func sqlWithUnsupportedParameterType() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.sql("SELECT ?", [1, 2])
+ }
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.sql("SELECT :x", args: ["x": [1, 2]])
+ }
+ await spark.stop()
+ }
+
@Test
func addInvalidArtifact() async throws {
await SparkSession.builder.clear()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]