This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new e0f67e2 [SPARK-58434] Support `pivot` in `GroupedData`
e0f67e2 is described below
commit e0f67e24937ae44068c85a7dff6794b26d058022
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 14:26:16 2026 -0700
[SPARK-58434] Support `pivot` in `GroupedData`
### What changes were proposed in this pull request?
This PR aims to support `pivot` in `GroupedData`.
```swift
let df2 = try await df.groupBy("year")
.pivot("course", ["dotNET", "Java"])
.agg("sum(earnings)")
```
If `values` is not given, the server computes the distinct values of the
pivot
column. Like the Scala client, `pivot` is only allowed after `groupBy` and
throws `SparkConnectError.UnsupportedOperation` otherwise.
### Why are the changes needed?
To support the standard `pivot` API like the Scala and Python clients.
### Does this PR introduce _any_ user-facing change?
No, this is a new API.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #454 from dongjoon-hyun/SPARK-58434.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Extension.swift | 90 +++++++++++++++-------------
Sources/SparkConnect/GroupedData.swift | 40 ++++++++++++-
Tests/SparkConnectTests/DataFrameTests.swift | 43 +++++++++++++
3 files changed, 132 insertions(+), 41 deletions(-)
diff --git a/Sources/SparkConnect/Extension.swift
b/Sources/SparkConnect/Extension.swift
index 32da34c..5684783 100644
--- a/Sources/SparkConnect/Extension.swift
+++ b/Sources/SparkConnect/Extension.swift
@@ -36,47 +36,8 @@ extension String {
}
private func toExpression(_ value: Sendable) throws ->
Spark_Connect_Expression {
- var literal = ExpressionLiteral()
- switch value {
- case let value as Bool:
- literal.boolean = value
- case let value as Int8:
- literal.byte = Int32(value)
- case let value as Int16:
- literal.short = Int32(value)
- case let value as Int32:
- literal.integer = value
- case let value as Int64:
- literal.long = value
- case let value as Int:
- literal.long = Int64(value)
- case let value as Float:
- literal.float = value
- case let value as Double:
- literal.double = value
- case let value as Decimal:
- var decimal = ExpressionLiteral.Decimal()
- decimal.value = "\(value)"
- literal.decimal = decimal
- case let value as String:
- literal.string = value
- case let value as Data:
- literal.binary = value
- case let value as Date:
- // `Date` is an absolute point in time, so it maps to Spark's `TIMESTAMP`
- // (microseconds since the UNIX epoch), not `DATE`.
- literal.timestamp = Int64(value.timeIntervalSince1970 * 1_000_000)
- default:
- if case Optional<Any>.none = value as Any {
- var dataType = DataType()
- dataType.null = DataType.NULL()
- literal.null = dataType
- } else {
- throw SparkConnectError.InvalidType
- }
- }
var expr = Spark_Connect_Expression()
- expr.literal = literal
+ expr.literal = try ExpressionLiteral(value)
return expr
}
@@ -199,6 +160,55 @@ extension String {
}
}
+/// Inside `extension ExpressionLiteral`, the unqualified name `Decimal`
refers to
+/// the nested `Literal.Decimal` proto type, so resolve Swift's `Decimal` at
file scope.
+private typealias SwiftDecimal = Decimal
+
+extension ExpressionLiteral {
+ /// Create an `ExpressionLiteral` from a Swift value of a supported type.
+ init(_ value: Sendable) throws {
+ self.init()
+ switch value {
+ case let value as Bool:
+ self.boolean = value
+ case let value as Int8:
+ self.byte = Int32(value)
+ case let value as Int16:
+ self.short = Int32(value)
+ case let value as Int32:
+ self.integer = value
+ case let value as Int64:
+ self.long = value
+ case let value as Int:
+ self.long = Int64(value)
+ case let value as Float:
+ self.float = value
+ case let value as Double:
+ self.double = value
+ case let value as SwiftDecimal:
+ var decimal = ExpressionLiteral.Decimal()
+ decimal.value = "\(value)"
+ self.decimal = decimal
+ case let value as String:
+ self.string = value
+ case let value as Data:
+ self.binary = value
+ case let value as Date:
+ // `Date` is an absolute point in time, so it maps to Spark's `TIMESTAMP`
+ // (microseconds since the UNIX epoch), not `DATE`.
+ self.timestamp = Int64(value.timeIntervalSince1970 * 1_000_000)
+ default:
+ if case Optional<Any>.none = value as Any {
+ var dataType = DataType()
+ dataType.null = DataType.NULL()
+ self.null = dataType
+ } else {
+ throw SparkConnectError.InvalidType
+ }
+ }
+ }
+}
+
extension [String: String] {
/// Get an array of `KeyValue` from `[String: String]`.
var toSparkConnectKeyValue: [KeyValue] {
diff --git a/Sources/SparkConnect/GroupedData.swift
b/Sources/SparkConnect/GroupedData.swift
index af2033b..cca3265 100644
--- a/Sources/SparkConnect/GroupedData.swift
+++ b/Sources/SparkConnect/GroupedData.swift
@@ -21,11 +21,46 @@ public actor GroupedData {
let df: DataFrame
let groupType: GroupType
let groupingCols: [String]
+ let pivot: Aggregate.Pivot?
- init(_ df: DataFrame, _ groupType: GroupType, _ groupingCols: [String]) {
+ init(
+ _ df: DataFrame, _ groupType: GroupType, _ groupingCols: [String],
+ _ pivot: Aggregate.Pivot? = nil
+ ) {
self.df = df
self.groupType = groupType
self.groupingCols = groupingCols
+ self.pivot = pivot
+ }
+
+ /// Pivots a column of the current ``DataFrame`` and performs the specified
aggregation.
+ /// The server computes the list of distinct values of the pivot column.
+ /// - Parameter colName: Name of the column to pivot.
+ /// - Returns: A ``GroupedData``.
+ public nonisolated func pivot(_ colName: String) throws -> GroupedData {
+ return try pivot(colName, [])
+ }
+
+ /// Pivots a column of the current ``DataFrame`` and performs the specified
aggregation.
+ ///
+ /// ```swift
+ /// let df2 = try await df.groupBy("year")
+ /// .pivot("course", ["dotNET", "Java"])
+ /// .agg("sum(earnings)")
+ /// ```
+ /// - Parameters:
+ /// - colName: Name of the column to pivot.
+ /// - values: List of values that will be translated to columns in the
output ``DataFrame``.
+ /// If empty, the server computes the list of distinct values of the
pivot column.
+ /// - Returns: A ``GroupedData``.
+ public nonisolated func pivot(_ colName: String, _ values: [Sendable])
throws -> GroupedData {
+ guard self.groupType == .groupby else {
+ throw SparkConnectError.UnsupportedOperation
+ }
+ var pivot = Aggregate.Pivot()
+ pivot.col = colName.toExpression
+ pivot.values = try values.map { try ExpressionLiteral($0) }
+ return GroupedData(self.df, GroupType.pivot, self.groupingCols, pivot)
}
public func agg(_ exprs: String...) async -> DataFrame {
@@ -105,6 +140,9 @@ public actor GroupedData {
aggregate.groupType = self.groupType
aggregate.groupingExpressions = self.groupingCols.map { $0.toExpression }
aggregate.aggregateExpressions = exprs
+ if let pivot = self.pivot {
+ aggregate.pivot = pivot
+ }
var relation = Relation()
relation.aggregate = aggregate
var plan = Plan()
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index f748d3a..a4a4e7b 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -1067,6 +1067,49 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func pivot() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.sql(DEALER_TABLE).groupBy("city")
+ .pivot("car_model", ["Honda Civic", "Honda CRV"])
+ .agg("sum(quantity)").orderBy("city").collect()
+ #expect(
+ rows == [
+ Row("Dublin", 20, 3),
+ Row("Fremont", 10, 7),
+ Row("San Jose", 5, nil),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func pivotWithoutValues() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await spark.sql(DEALER_TABLE).groupBy("city")
+ .pivot("car_model")
+ .agg("sum(quantity)").orderBy("city").collect()
+ #expect(
+ rows == [
+ Row("Dublin", 10, 3, 20),
+ Row("Fremont", 15, 7, 10),
+ Row("San Jose", 8, nil, 5),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func pivotUnsupportedOperation() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(DEALER_TABLE)
+ await #expect(throws: SparkConnectError.UnsupportedOperation) {
+ try await df.rollup("city").pivot("car_model")
+ }
+ await #expect(throws: SparkConnectError.UnsupportedOperation) {
+ try await df.cube("city").pivot("car_model")
+ }
+ await spark.stop()
+ }
+
@Test
func toJSON() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]