This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new e0f67e2  [SPARK-58434] Support `pivot` in `GroupedData`
e0f67e2 is described below

commit e0f67e24937ae44068c85a7dff6794b26d058022
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 14:26:16 2026 -0700

    [SPARK-58434] Support `pivot` in `GroupedData`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `pivot` in `GroupedData`.
    
    ```swift
    let df2 = try await df.groupBy("year")
        .pivot("course", ["dotNET", "Java"])
        .agg("sum(earnings)")
    ```
    
    If `values` is not given, the server computes the distinct values of the 
pivot
    column. Like the Scala client, `pivot` is only allowed after `groupBy` and
    throws `SparkConnectError.UnsupportedOperation` otherwise.
    
    ### Why are the changes needed?
    
    To support the standard `pivot` API like the Scala and Python clients.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this is a new API.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #454 from dongjoon-hyun/SPARK-58434.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/Extension.swift         | 90 +++++++++++++++-------------
 Sources/SparkConnect/GroupedData.swift       | 40 ++++++++++++-
 Tests/SparkConnectTests/DataFrameTests.swift | 43 +++++++++++++
 3 files changed, 132 insertions(+), 41 deletions(-)

diff --git a/Sources/SparkConnect/Extension.swift 
b/Sources/SparkConnect/Extension.swift
index 32da34c..5684783 100644
--- a/Sources/SparkConnect/Extension.swift
+++ b/Sources/SparkConnect/Extension.swift
@@ -36,47 +36,8 @@ extension String {
   }
 
   private func toExpression(_ value: Sendable) throws -> 
Spark_Connect_Expression {
-    var literal = ExpressionLiteral()
-    switch value {
-    case let value as Bool:
-      literal.boolean = value
-    case let value as Int8:
-      literal.byte = Int32(value)
-    case let value as Int16:
-      literal.short = Int32(value)
-    case let value as Int32:
-      literal.integer = value
-    case let value as Int64:
-      literal.long = value
-    case let value as Int:
-      literal.long = Int64(value)
-    case let value as Float:
-      literal.float = value
-    case let value as Double:
-      literal.double = value
-    case let value as Decimal:
-      var decimal = ExpressionLiteral.Decimal()
-      decimal.value = "\(value)"
-      literal.decimal = decimal
-    case let value as String:
-      literal.string = value
-    case let value as Data:
-      literal.binary = value
-    case let value as Date:
-      // `Date` is an absolute point in time, so it maps to Spark's `TIMESTAMP`
-      // (microseconds since the UNIX epoch), not `DATE`.
-      literal.timestamp = Int64(value.timeIntervalSince1970 * 1_000_000)
-    default:
-      if case Optional<Any>.none = value as Any {
-        var dataType = DataType()
-        dataType.null = DataType.NULL()
-        literal.null = dataType
-      } else {
-        throw SparkConnectError.InvalidType
-      }
-    }
     var expr = Spark_Connect_Expression()
-    expr.literal = literal
+    expr.literal = try ExpressionLiteral(value)
     return expr
   }
 
@@ -199,6 +160,55 @@ extension String {
   }
 }
 
+/// Inside `extension ExpressionLiteral`, the unqualified name `Decimal` 
refers to
+/// the nested `Literal.Decimal` proto type, so resolve Swift's `Decimal` at 
file scope.
+private typealias SwiftDecimal = Decimal
+
+extension ExpressionLiteral {
+  /// Create an `ExpressionLiteral` from a Swift value of a supported type.
+  init(_ value: Sendable) throws {
+    self.init()
+    switch value {
+    case let value as Bool:
+      self.boolean = value
+    case let value as Int8:
+      self.byte = Int32(value)
+    case let value as Int16:
+      self.short = Int32(value)
+    case let value as Int32:
+      self.integer = value
+    case let value as Int64:
+      self.long = value
+    case let value as Int:
+      self.long = Int64(value)
+    case let value as Float:
+      self.float = value
+    case let value as Double:
+      self.double = value
+    case let value as SwiftDecimal:
+      var decimal = ExpressionLiteral.Decimal()
+      decimal.value = "\(value)"
+      self.decimal = decimal
+    case let value as String:
+      self.string = value
+    case let value as Data:
+      self.binary = value
+    case let value as Date:
+      // `Date` is an absolute point in time, so it maps to Spark's `TIMESTAMP`
+      // (microseconds since the UNIX epoch), not `DATE`.
+      self.timestamp = Int64(value.timeIntervalSince1970 * 1_000_000)
+    default:
+      if case Optional<Any>.none = value as Any {
+        var dataType = DataType()
+        dataType.null = DataType.NULL()
+        self.null = dataType
+      } else {
+        throw SparkConnectError.InvalidType
+      }
+    }
+  }
+}
+
 extension [String: String] {
   /// Get an array of `KeyValue` from `[String: String]`.
   var toSparkConnectKeyValue: [KeyValue] {
diff --git a/Sources/SparkConnect/GroupedData.swift 
b/Sources/SparkConnect/GroupedData.swift
index af2033b..cca3265 100644
--- a/Sources/SparkConnect/GroupedData.swift
+++ b/Sources/SparkConnect/GroupedData.swift
@@ -21,11 +21,46 @@ public actor GroupedData {
   let df: DataFrame
   let groupType: GroupType
   let groupingCols: [String]
+  let pivot: Aggregate.Pivot?
 
-  init(_ df: DataFrame, _ groupType: GroupType, _ groupingCols: [String]) {
+  init(
+    _ df: DataFrame, _ groupType: GroupType, _ groupingCols: [String],
+    _ pivot: Aggregate.Pivot? = nil
+  ) {
     self.df = df
     self.groupType = groupType
     self.groupingCols = groupingCols
+    self.pivot = pivot
+  }
+
+  /// Pivots a column of the current ``DataFrame`` and performs the specified 
aggregation.
+  /// The server computes the list of distinct values of the pivot column.
+  /// - Parameter colName: Name of the column to pivot.
+  /// - Returns: A ``GroupedData``.
+  public nonisolated func pivot(_ colName: String) throws -> GroupedData {
+    return try pivot(colName, [])
+  }
+
+  /// Pivots a column of the current ``DataFrame`` and performs the specified 
aggregation.
+  ///
+  /// ```swift
+  /// let df2 = try await df.groupBy("year")
+  ///     .pivot("course", ["dotNET", "Java"])
+  ///     .agg("sum(earnings)")
+  /// ```
+  /// - Parameters:
+  ///   - colName: Name of the column to pivot.
+  ///   - values: List of values that will be translated to columns in the 
output ``DataFrame``.
+  ///     If empty, the server computes the list of distinct values of the 
pivot column.
+  /// - Returns: A ``GroupedData``.
+  public nonisolated func pivot(_ colName: String, _ values: [Sendable]) 
throws -> GroupedData {
+    guard self.groupType == .groupby else {
+      throw SparkConnectError.UnsupportedOperation
+    }
+    var pivot = Aggregate.Pivot()
+    pivot.col = colName.toExpression
+    pivot.values = try values.map { try ExpressionLiteral($0) }
+    return GroupedData(self.df, GroupType.pivot, self.groupingCols, pivot)
   }
 
   public func agg(_ exprs: String...) async -> DataFrame {
@@ -105,6 +140,9 @@ public actor GroupedData {
     aggregate.groupType = self.groupType
     aggregate.groupingExpressions = self.groupingCols.map { $0.toExpression }
     aggregate.aggregateExpressions = exprs
+    if let pivot = self.pivot {
+      aggregate.pivot = pivot
+    }
     var relation = Relation()
     relation.aggregate = aggregate
     var plan = Plan()
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index f748d3a..a4a4e7b 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -1067,6 +1067,49 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func pivot() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await spark.sql(DEALER_TABLE).groupBy("city")
+      .pivot("car_model", ["Honda Civic", "Honda CRV"])
+      .agg("sum(quantity)").orderBy("city").collect()
+    #expect(
+      rows == [
+        Row("Dublin", 20, 3),
+        Row("Fremont", 10, 7),
+        Row("San Jose", 5, nil),
+      ])
+    await spark.stop()
+  }
+
+  @Test
+  func pivotWithoutValues() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await spark.sql(DEALER_TABLE).groupBy("city")
+      .pivot("car_model")
+      .agg("sum(quantity)").orderBy("city").collect()
+    #expect(
+      rows == [
+        Row("Dublin", 10, 3, 20),
+        Row("Fremont", 15, 7, 10),
+        Row("San Jose", 8, nil, 5),
+      ])
+    await spark.stop()
+  }
+
+  @Test
+  func pivotUnsupportedOperation() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql(DEALER_TABLE)
+    await #expect(throws: SparkConnectError.UnsupportedOperation) {
+      try await df.rollup("city").pivot("car_model")
+    }
+    await #expect(throws: SparkConnectError.UnsupportedOperation) {
+      try await df.cube("city").pivot("car_model")
+    }
+    await spark.stop()
+  }
+
   @Test
   func toJSON() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to