This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 09a1869  [SPARK-58424] Add `count`/`sum`/`avg`/`mean`/`min`/`max` to 
`GroupedData`
09a1869 is described below

commit 09a18698560805b97b4e3f64cce9e8b3a6a4a3e3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 13:04:33 2026 -0700

    [SPARK-58424] Add `count`/`sum`/`avg`/`mean`/`min`/`max` to `GroupedData`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to add shortcut aggregation methods to `GroupedData`:
    
    - `count()` — counts the number of records for each group with a `count` 
result column.
    - `sum(_ colNames: String...)`
    - `avg(_ colNames: String...)`
    - `mean(_ colNames: String...)` (an alias of `avg`)
    - `min(_ colNames: String...)`
    - `max(_ colNames: String...)`
    
    The semantics follow the Scala Spark Connect client's 
`RelationalGroupedDataset`:
    `count()` is built as `count(lit(1)).alias("count")`, and the 
column-name-based
    methods map each name via the corresponding `Column` function without 
client-side
    numeric-type validation (delegated to the server, like the Connect Scala 
client).
    All methods reuse the existing `Aggregate` relation path, so they work 
identically
    for `groupBy`, `rollup`, and `cube`.
    
    ```swift
    let df = try await spark.sql(DEALER_TABLE).groupBy("city").count()
    let df2 = try await spark.sql(DEALER_TABLE).rollup("city").sum("quantity")
    ```
    
    ### Why are the changes needed?
    
    Previously, `GroupedData` only provided `agg`. These shortcut methods 
provide
    feature parity with PySpark's `GroupedData` and the Scala client's
    `RelationalGroupedDataset` for the most common aggregations.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a new API addition.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #448 from dongjoon-hyun/SPARK-58424.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/GroupedData.swift       | 53 +++++++++++++++++++++
 Tests/SparkConnectTests/DataFrameTests.swift | 69 ++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)

diff --git a/Sources/SparkConnect/GroupedData.swift 
b/Sources/SparkConnect/GroupedData.swift
index b4a2467..af2033b 100644
--- a/Sources/SparkConnect/GroupedData.swift
+++ b/Sources/SparkConnect/GroupedData.swift
@@ -46,6 +46,59 @@ public actor GroupedData {
     return await buildAggregate(([expr] + exprs).map { $0.expr })
   }
 
+  /// Counts the number of records for each group. The resulting ``DataFrame`` 
will also contain
+  /// the grouping columns and a `count` column.
+  /// - Returns: A ``DataFrame``.
+  public func count() async -> DataFrame {
+    return await 
buildAggregate([SparkConnect.count(lit(Int32(1))).alias("count").expr])
+  }
+
+  /// Computes the sum for each numeric column for each group. The resulting 
``DataFrame`` will
+  /// also contain the grouping columns.
+  /// - Parameter colNames: Column names to aggregate.
+  /// - Returns: A ``DataFrame``.
+  public func sum(_ colNames: String...) async -> DataFrame {
+    return await aggregateColumns(colNames, SparkConnect.sum)
+  }
+
+  /// Computes the average for each numeric column for each group. The 
resulting ``DataFrame``
+  /// will also contain the grouping columns.
+  /// - Parameter colNames: Column names to aggregate.
+  /// - Returns: A ``DataFrame``.
+  public func avg(_ colNames: String...) async -> DataFrame {
+    return await aggregateColumns(colNames, SparkConnect.avg)
+  }
+
+  /// Computes the average for each numeric column for each group. This is an 
alias of `avg`.
+  /// The resulting ``DataFrame`` will also contain the grouping columns.
+  /// - Parameter colNames: Column names to aggregate.
+  /// - Returns: A ``DataFrame``.
+  public func mean(_ colNames: String...) async -> DataFrame {
+    return await aggregateColumns(colNames, SparkConnect.avg)
+  }
+
+  /// Computes the min value for each numeric column for each group. The 
resulting ``DataFrame``
+  /// will also contain the grouping columns.
+  /// - Parameter colNames: Column names to aggregate.
+  /// - Returns: A ``DataFrame``.
+  public func min(_ colNames: String...) async -> DataFrame {
+    return await aggregateColumns(colNames, SparkConnect.min)
+  }
+
+  /// Computes the max value for each numeric column for each group. The 
resulting ``DataFrame``
+  /// will also contain the grouping columns.
+  /// - Parameter colNames: Column names to aggregate.
+  /// - Returns: A ``DataFrame``.
+  public func max(_ colNames: String...) async -> DataFrame {
+    return await aggregateColumns(colNames, SparkConnect.max)
+  }
+
+  private func aggregateColumns(
+    _ colNames: [String], _ function: (Column) -> Column
+  ) async -> DataFrame {
+    return await buildAggregate(colNames.map { function(col($0)).expr })
+  }
+
   private func buildAggregate(_ exprs: [Spark_Connect_Expression]) async -> 
DataFrame {
     var aggregate = Aggregate()
     aggregate.input = await (self.df.getPlan() as! Plan).root
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index e052095..f417d13 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -881,6 +881,75 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func groupByCount() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await 
spark.sql(DEALER_TABLE).groupBy("city").count().orderBy("city")
+    #expect(try await df.columns == ["city", "count"])
+    #expect(
+      try await df.collect() == [Row("Dublin", 3), Row("Fremont", 3), Row("San 
Jose", 2)])
+    await spark.stop()
+  }
+
+  @Test
+  func groupBySum() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql(DEALER_TABLE).groupBy("city").sum("quantity", 
"id")
+      .orderBy("city")
+    #expect(try await df.columns == ["city", "sum(quantity)", "sum(id)"])
+    #expect(
+      try await df.collect() == [
+        Row("Dublin", 33, 600), Row("Fremont", 32, 300), Row("San Jose", 13, 
600),
+      ])
+    await spark.stop()
+  }
+
+  @Test
+  func groupByAvg() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let expected = [Row("Dublin", 200.0), Row("Fremont", 100.0), Row("San 
Jose", 300.0)]
+    let df = try await 
spark.sql(DEALER_TABLE).groupBy("city").avg("id").orderBy("city")
+    #expect(try await df.columns == ["city", "avg(id)"])
+    #expect(try await df.collect() == expected)
+    let df2 = try await 
spark.sql(DEALER_TABLE).groupBy("city").mean("id").orderBy("city")
+    #expect(try await df2.columns == ["city", "avg(id)"])
+    #expect(try await df2.collect() == expected)
+    await spark.stop()
+  }
+
+  @Test
+  func groupByMinMax() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await 
spark.sql(DEALER_TABLE).groupBy("city").min("quantity").orderBy("city")
+    #expect(try await df.columns == ["city", "min(quantity)"])
+    #expect(
+      try await df.collect() == [Row("Dublin", 3), Row("Fremont", 7), Row("San 
Jose", 5)])
+    let df2 = try await 
spark.sql(DEALER_TABLE).groupBy("city").max("quantity").orderBy("city")
+    #expect(try await df2.columns == ["city", "max(quantity)"])
+    #expect(
+      try await df2.collect() == [Row("Dublin", 20), Row("Fremont", 15), 
Row("San Jose", 8)])
+    await spark.stop()
+  }
+
+  @Test
+  func rollupCount() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await 
spark.sql(DEALER_TABLE).rollup("city").count().orderBy("city").collect()
+    #expect(
+      rows == [Row("Dublin", 3), Row("Fremont", 3), Row("San Jose", 2), 
Row(nil, 8)])
+    await spark.stop()
+  }
+
+  @Test
+  func cubeSum() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let rows = try await 
spark.sql(DEALER_TABLE).cube("city").sum("quantity").orderBy("city")
+      .collect()
+    #expect(
+      rows == [Row("Dublin", 33), Row("Fremont", 32), Row("San Jose", 13), 
Row(nil, 78)])
+    await spark.stop()
+  }
+
   @Test
   func rollup() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to