This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 09a1869 [SPARK-58424] Add `count`/`sum`/`avg`/`mean`/`min`/`max` to
`GroupedData`
09a1869 is described below
commit 09a18698560805b97b4e3f64cce9e8b3a6a4a3e3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 13:04:33 2026 -0700
[SPARK-58424] Add `count`/`sum`/`avg`/`mean`/`min`/`max` to `GroupedData`
### What changes were proposed in this pull request?
This PR aims to add shortcut aggregation methods to `GroupedData`:
- `count()` — counts the number of records for each group with a `count`
result column.
- `sum(_ colNames: String...)`
- `avg(_ colNames: String...)`
- `mean(_ colNames: String...)` (an alias of `avg`)
- `min(_ colNames: String...)`
- `max(_ colNames: String...)`
The semantics follow the Scala Spark Connect client's
`RelationalGroupedDataset`:
`count()` is built as `count(lit(1)).alias("count")`, and the
column-name-based
methods map each name via the corresponding `Column` function without
client-side
numeric-type validation (delegated to the server, like the Connect Scala
client).
All methods reuse the existing `Aggregate` relation path, so they work
identically
for `groupBy`, `rollup`, and `cube`.
```swift
let df = try await spark.sql(DEALER_TABLE).groupBy("city").count()
let df2 = try await spark.sql(DEALER_TABLE).rollup("city").sum("quantity")
```
### Why are the changes needed?
Previously, `GroupedData` only provided `agg`. These shortcut methods
provide
feature parity with PySpark's `GroupedData` and the Scala client's
`RelationalGroupedDataset` for the most common aggregations.
### Does this PR introduce _any_ user-facing change?
No. This is a new API addition.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #448 from dongjoon-hyun/SPARK-58424.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/GroupedData.swift | 53 +++++++++++++++++++++
Tests/SparkConnectTests/DataFrameTests.swift | 69 ++++++++++++++++++++++++++++
2 files changed, 122 insertions(+)
diff --git a/Sources/SparkConnect/GroupedData.swift
b/Sources/SparkConnect/GroupedData.swift
index b4a2467..af2033b 100644
--- a/Sources/SparkConnect/GroupedData.swift
+++ b/Sources/SparkConnect/GroupedData.swift
@@ -46,6 +46,59 @@ public actor GroupedData {
return await buildAggregate(([expr] + exprs).map { $0.expr })
}
+ /// Counts the number of records for each group. The resulting ``DataFrame``
will also contain
+ /// the grouping columns and a `count` column.
+ /// - Returns: A ``DataFrame``.
+ public func count() async -> DataFrame {
+ return await
buildAggregate([SparkConnect.count(lit(Int32(1))).alias("count").expr])
+ }
+
+ /// Computes the sum for each numeric column for each group. The resulting
``DataFrame`` will
+ /// also contain the grouping columns.
+ /// - Parameter colNames: Column names to aggregate.
+ /// - Returns: A ``DataFrame``.
+ public func sum(_ colNames: String...) async -> DataFrame {
+ return await aggregateColumns(colNames, SparkConnect.sum)
+ }
+
+ /// Computes the average for each numeric column for each group. The
resulting ``DataFrame``
+ /// will also contain the grouping columns.
+ /// - Parameter colNames: Column names to aggregate.
+ /// - Returns: A ``DataFrame``.
+ public func avg(_ colNames: String...) async -> DataFrame {
+ return await aggregateColumns(colNames, SparkConnect.avg)
+ }
+
+ /// Computes the average for each numeric column for each group. This is an
alias of `avg`.
+ /// The resulting ``DataFrame`` will also contain the grouping columns.
+ /// - Parameter colNames: Column names to aggregate.
+ /// - Returns: A ``DataFrame``.
+ public func mean(_ colNames: String...) async -> DataFrame {
+ return await aggregateColumns(colNames, SparkConnect.avg)
+ }
+
+ /// Computes the min value for each numeric column for each group. The
resulting ``DataFrame``
+ /// will also contain the grouping columns.
+ /// - Parameter colNames: Column names to aggregate.
+ /// - Returns: A ``DataFrame``.
+ public func min(_ colNames: String...) async -> DataFrame {
+ return await aggregateColumns(colNames, SparkConnect.min)
+ }
+
+ /// Computes the max value for each numeric column for each group. The
resulting ``DataFrame``
+ /// will also contain the grouping columns.
+ /// - Parameter colNames: Column names to aggregate.
+ /// - Returns: A ``DataFrame``.
+ public func max(_ colNames: String...) async -> DataFrame {
+ return await aggregateColumns(colNames, SparkConnect.max)
+ }
+
+ private func aggregateColumns(
+ _ colNames: [String], _ function: (Column) -> Column
+ ) async -> DataFrame {
+ return await buildAggregate(colNames.map { function(col($0)).expr })
+ }
+
private func buildAggregate(_ exprs: [Spark_Connect_Expression]) async ->
DataFrame {
var aggregate = Aggregate()
aggregate.input = await (self.df.getPlan() as! Plan).root
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index e052095..f417d13 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -881,6 +881,75 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func groupByCount() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await
spark.sql(DEALER_TABLE).groupBy("city").count().orderBy("city")
+ #expect(try await df.columns == ["city", "count"])
+ #expect(
+ try await df.collect() == [Row("Dublin", 3), Row("Fremont", 3), Row("San
Jose", 2)])
+ await spark.stop()
+ }
+
+ @Test
+ func groupBySum() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(DEALER_TABLE).groupBy("city").sum("quantity",
"id")
+ .orderBy("city")
+ #expect(try await df.columns == ["city", "sum(quantity)", "sum(id)"])
+ #expect(
+ try await df.collect() == [
+ Row("Dublin", 33, 600), Row("Fremont", 32, 300), Row("San Jose", 13,
600),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func groupByAvg() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let expected = [Row("Dublin", 200.0), Row("Fremont", 100.0), Row("San
Jose", 300.0)]
+ let df = try await
spark.sql(DEALER_TABLE).groupBy("city").avg("id").orderBy("city")
+ #expect(try await df.columns == ["city", "avg(id)"])
+ #expect(try await df.collect() == expected)
+ let df2 = try await
spark.sql(DEALER_TABLE).groupBy("city").mean("id").orderBy("city")
+ #expect(try await df2.columns == ["city", "avg(id)"])
+ #expect(try await df2.collect() == expected)
+ await spark.stop()
+ }
+
+ @Test
+ func groupByMinMax() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await
spark.sql(DEALER_TABLE).groupBy("city").min("quantity").orderBy("city")
+ #expect(try await df.columns == ["city", "min(quantity)"])
+ #expect(
+ try await df.collect() == [Row("Dublin", 3), Row("Fremont", 7), Row("San
Jose", 5)])
+ let df2 = try await
spark.sql(DEALER_TABLE).groupBy("city").max("quantity").orderBy("city")
+ #expect(try await df2.columns == ["city", "max(quantity)"])
+ #expect(
+ try await df2.collect() == [Row("Dublin", 20), Row("Fremont", 15),
Row("San Jose", 8)])
+ await spark.stop()
+ }
+
+ @Test
+ func rollupCount() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await
spark.sql(DEALER_TABLE).rollup("city").count().orderBy("city").collect()
+ #expect(
+ rows == [Row("Dublin", 3), Row("Fremont", 3), Row("San Jose", 2),
Row(nil, 8)])
+ await spark.stop()
+ }
+
+ @Test
+ func cubeSum() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let rows = try await
spark.sql(DEALER_TABLE).cube("city").sum("quantity").orderBy("city")
+ .collect()
+ #expect(
+ rows == [Row("Dublin", 33), Row("Fremont", 32), Row("San Jose", 13),
Row(nil, 78)])
+ await spark.stop()
+ }
+
@Test
func rollup() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]