This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 765f921 [SPARK-58492] Support window functions with `Window`,
`WindowSpec` and `Column.over`
765f921 is described below
commit 765f9210d0715e8457b4ab407628cc4eb81dcec3
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 20:48:35 2026 -0700
[SPARK-58492] Support window functions with `Window`, `WindowSpec` and
`Column.over`
### What changes were proposed in this pull request?
This PR aims to support window functions in `SparkConnect` module.
- New `Window` and `WindowSpec` types with `partitionBy`, `orderBy`,
`rowsBetween`, `rangeBetween`, and the
`unboundedPreceding`/`unboundedFollowing`/`currentRow` constants.
- New `Column.over(_:)` and `Column.over()` building `Expression.Window`.
- New window functions in `WindowFunctions.swift`: `row_number`, `rank`,
`dense_rank`, `percent_rank`, `ntile`, `cume_dist`, `lag`, `lead`, `nth_value`.
```swift
let window = Window.partitionBy("dept").orderBy(col("salary").desc())
df.withColumn("rank", rank().over(window))
```
### Why are the changes needed?
To provide window function support like the Scala/PySpark clients.
### Does this PR introduce _any_ user-facing change?
No. This is a new feature addition.
### How was this patch tested?
Pass the CIs with the newly added `WindowTests` test suite.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #465 from dongjoon-hyun/SPARK-58492.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Column.swift | 31 +++++
Sources/SparkConnect/Window.swift | 99 ++++++++++++++++
Sources/SparkConnect/WindowFunctions.swift | 120 +++++++++++++++++++
Sources/SparkConnect/WindowSpec.swift | 175 ++++++++++++++++++++++++++++
Tests/SparkConnectTests/WindowTests.swift | 179 +++++++++++++++++++++++++++++
5 files changed, 604 insertions(+)
diff --git a/Sources/SparkConnect/Column.swift
b/Sources/SparkConnect/Column.swift
index 5698436..32665a2 100644
--- a/Sources/SparkConnect/Column.swift
+++ b/Sources/SparkConnect/Column.swift
@@ -86,6 +86,37 @@ public struct Column: Sendable {
return Column(expr)
}
+ /// Defines a windowing column.
+ ///
+ /// ```swift
+ /// df.select(rank().over(Window.partitionBy("dept").orderBy("salary")))
+ /// ```
+ /// - Parameter window: A ``WindowSpec`` defining the partitioning,
ordering, and frame.
+ /// - Returns: A ``Column``.
+ public func over(_ window: WindowSpec) -> Column {
+ var win = Spark_Connect_Expression.Window()
+ win.windowFunction = self.expr
+ win.partitionSpec = window.partitionSpec
+ win.orderSpec = window.orderSpec
+ if let frame = window.frame {
+ win.frameSpec = frame
+ }
+ var expr = Spark_Connect_Expression()
+ expr.window = win
+ return Column(expr)
+ }
+
+ /// Defines an empty analytic clause. In this case the analytic function is
applied and
+ /// presented for all rows in the result set.
+ ///
+ /// ```swift
+ /// df.select(sum(col("price")).over())
+ /// ```
+ /// - Returns: A ``Column``.
+ public func over() -> Column {
+ return over(WindowSpec())
+ }
+
// MARK: - Predicates
/// Returns an expression that is true if this column is null.
diff --git a/Sources/SparkConnect/Window.swift
b/Sources/SparkConnect/Window.swift
new file mode 100644
index 0000000..7e98514
--- /dev/null
+++ b/Sources/SparkConnect/Window.swift
@@ -0,0 +1,99 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+/// Utility functions for defining window in ``DataFrame``s.
+///
+/// ```swift
+/// // PARTITION BY country ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND
CURRENT ROW
+/// Window.partitionBy("country").orderBy("date")
+/// .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+///
+/// // PARTITION BY country ORDER BY date ROWS BETWEEN 3 PRECEDING AND 3
FOLLOWING
+/// Window.partitionBy("country").orderBy("date").rowsBetween(-3, 3)
+/// ```
+///
+/// When ordering is not defined, an unbounded window frame (rowFrame,
unboundedPreceding,
+/// unboundedFollowing) is used by default. When ordering is defined, a
growing window frame
+/// (rangeFrame, unboundedPreceding, currentRow) is used by default.
+public struct Window: Sendable {
+ private init() {}
+
+ /// Value representing the first row in the partition, equivalent to
"UNBOUNDED PRECEDING" in
+ /// SQL. This can be used to specify the frame boundaries.
+ public static let unboundedPreceding: Int64 = Int64.min
+
+ /// Value representing the last row in the partition, equivalent to
"UNBOUNDED FOLLOWING" in
+ /// SQL. This can be used to specify the frame boundaries.
+ public static let unboundedFollowing: Int64 = Int64.max
+
+ /// Value representing the current row. This can be used to specify the
frame boundaries.
+ public static let currentRow: Int64 = 0
+
+ /// Creates a ``WindowSpec`` with the partitioning defined.
+ /// - Parameter colNames: Column names to partition by.
+ /// - Returns: A ``WindowSpec``.
+ public static func partitionBy(_ colNames: String...) -> WindowSpec {
+ return WindowSpec().partitionBy(colNames.map { Column($0) })
+ }
+
+ /// Creates a ``WindowSpec`` with the partitioning defined.
+ /// - Parameter cols: ``Column``s to partition by.
+ /// - Returns: A ``WindowSpec``.
+ public static func partitionBy(_ cols: Column...) -> WindowSpec {
+ return WindowSpec().partitionBy(cols)
+ }
+
+ /// Creates a ``WindowSpec`` with the ordering defined.
+ /// - Parameter colNames: Column names to order by.
+ /// - Returns: A ``WindowSpec``.
+ public static func orderBy(_ colNames: String...) -> WindowSpec {
+ return WindowSpec().orderBy(colNames.map { Column($0) })
+ }
+
+ /// Creates a ``WindowSpec`` with the ordering defined.
+ /// - Parameter cols: ``Column``s or sort expressions like `col("a").desc()`
to order by.
+ /// - Returns: A ``WindowSpec``.
+ public static func orderBy(_ cols: Column...) -> WindowSpec {
+ return WindowSpec().orderBy(cols)
+ }
+
+ /// Creates a ``WindowSpec`` with the frame boundaries defined, from `start`
(inclusive) to
+ /// `end` (inclusive). See ``WindowSpec/rowsBetween(_:_:)`` for details.
+ /// - Parameters:
+ /// - start: A boundary start, inclusive. The frame is unbounded if this is
+ /// ``unboundedPreceding``.
+ /// - end: A boundary end, inclusive. The frame is unbounded if this is
+ /// ``unboundedFollowing``.
+ /// - Returns: A ``WindowSpec``.
+ public static func rowsBetween(_ start: Int64, _ end: Int64) throws ->
WindowSpec {
+ return try WindowSpec().rowsBetween(start, end)
+ }
+
+ /// Creates a ``WindowSpec`` with the frame boundaries defined, from `start`
(inclusive) to
+ /// `end` (inclusive). See ``WindowSpec/rangeBetween(_:_:)`` for details.
+ /// - Parameters:
+ /// - start: A boundary start, inclusive. The frame is unbounded if this is
+ /// ``unboundedPreceding``.
+ /// - end: A boundary end, inclusive. The frame is unbounded if this is
+ /// ``unboundedFollowing``.
+ /// - Returns: A ``WindowSpec``.
+ public static func rangeBetween(_ start: Int64, _ end: Int64) -> WindowSpec {
+ return WindowSpec().rangeBetween(start, end)
+ }
+}
diff --git a/Sources/SparkConnect/WindowFunctions.swift
b/Sources/SparkConnect/WindowFunctions.swift
new file mode 100644
index 0000000..2cb1a10
--- /dev/null
+++ b/Sources/SparkConnect/WindowFunctions.swift
@@ -0,0 +1,120 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+/// Returns the cumulative distribution of values within a window partition,
i.e. the fraction
+/// of rows that are below the current row.
+/// - Returns: A ``Column``.
+public func cume_dist() -> Column {
+ return fn("cume_dist")
+}
+
+/// Returns the rank of rows within a window partition, without any gaps.
+///
+/// The difference between ``rank()`` and ``dense_rank()`` is that
`dense_rank` leaves no gaps in
+/// ranking sequence when there are ties. That is, if you were ranking a
competition using
+/// `dense_rank` and had three people tie for second place, you would say that
all three were in
+/// second place and that the next person came in third.
+/// - Returns: A ``Column``.
+public func dense_rank() -> Column {
+ return fn("dense_rank")
+}
+
+/// Returns the value that is `offset` rows before the current row, and null
if there is less
+/// than `offset` rows before the current row.
+/// - Parameters:
+/// - e: A ``Column`` to get the value of.
+/// - offset: A number of rows back from the current row from which to
obtain a value.
+/// - Returns: A ``Column``.
+public func lag(_ e: Column, _ offset: Int32) -> Column {
+ return fn("lag", e, lit(offset))
+}
+
+/// Returns the value that is `offset` rows before the current row, and
`defaultValue` if there
+/// is less than `offset` rows before the current row.
+/// - Parameters:
+/// - e: A ``Column`` to get the value of.
+/// - offset: A number of rows back from the current row from which to
obtain a value.
+/// - defaultValue: A default literal value.
+/// - Returns: A ``Column``.
+public func lag(_ e: Column, _ offset: Int32, _ defaultValue: some
SparkLiteral) -> Column {
+ return fn("lag", e, lit(offset), defaultValue.toLiteralColumn)
+}
+
+/// Returns the value that is `offset` rows after the current row, and null if
there is less
+/// than `offset` rows after the current row.
+/// - Parameters:
+/// - e: A ``Column`` to get the value of.
+/// - offset: A number of rows after the current row from which to obtain a
value.
+/// - Returns: A ``Column``.
+public func lead(_ e: Column, _ offset: Int32) -> Column {
+ return fn("lead", e, lit(offset))
+}
+
+/// Returns the value that is `offset` rows after the current row, and
`defaultValue` if there
+/// is less than `offset` rows after the current row.
+/// - Parameters:
+/// - e: A ``Column`` to get the value of.
+/// - offset: A number of rows after the current row from which to obtain a
value.
+/// - defaultValue: A default literal value.
+/// - Returns: A ``Column``.
+public func lead(_ e: Column, _ offset: Int32, _ defaultValue: some
SparkLiteral) -> Column {
+ return fn("lead", e, lit(offset), defaultValue.toLiteralColumn)
+}
+
+/// Returns the value that is the `offset`th row of the window frame (counting
from 1), and null
+/// if the size of window frame is less than `offset` rows.
+/// - Parameters:
+/// - e: A ``Column`` to get the value of.
+/// - offset: A position of the row within the window frame, counting from 1.
+/// - Returns: A ``Column``.
+public func nth_value(_ e: Column, _ offset: Int32) -> Column {
+ return fn("nth_value", e, lit(offset))
+}
+
+/// Returns the ntile group id (from 1 to `n` inclusive) in an ordered window
partition. For
+/// example, if `n` is 4, the first quarter of the rows will get value 1, the
second quarter will
+/// get 2, the third quarter will get 3, and the last quarter will get 4.
+/// - Parameter n: A number of buckets.
+/// - Returns: A ``Column``.
+public func ntile(_ n: Int32) -> Column {
+ return fn("ntile", lit(n))
+}
+
+/// Returns the relative rank (i.e. percentile) of rows within a window
partition.
+/// - Returns: A ``Column``.
+public func percent_rank() -> Column {
+ return fn("percent_rank")
+}
+
+/// Returns the rank of rows within a window partition.
+///
+/// The difference between ``rank()`` and ``dense_rank()`` is that
`dense_rank` leaves no gaps in
+/// ranking sequence when there are ties. That is, if you were ranking a
competition using
+/// `dense_rank` and had three people tie for second place, you would say that
all three were in
+/// second place and that the next person came in third.
+/// - Returns: A ``Column``.
+public func rank() -> Column {
+ return fn("rank")
+}
+
+/// Returns a sequential number starting at 1 within a window partition.
+/// - Returns: A ``Column``.
+public func row_number() -> Column {
+ return fn("row_number")
+}
diff --git a/Sources/SparkConnect/WindowSpec.swift
b/Sources/SparkConnect/WindowSpec.swift
new file mode 100644
index 0000000..936a38a
--- /dev/null
+++ b/Sources/SparkConnect/WindowSpec.swift
@@ -0,0 +1,175 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+/// A window specification that defines the partitioning, ordering, and frame
boundaries.
+///
+/// Use the static methods in ``Window`` to create a ``WindowSpec``.
+///
+/// ```swift
+/// df.withColumn("rank",
rank().over(Window.partitionBy("dept").orderBy("salary")))
+/// ```
+public struct WindowSpec: Sendable {
+ let partitionSpec: [Spark_Connect_Expression]
+ let orderSpec: [Spark_Connect_Expression.SortOrder]
+ let frame: Spark_Connect_Expression.Window.WindowFrame?
+
+ init(
+ _ partitionSpec: [Spark_Connect_Expression] = [],
+ _ orderSpec: [Spark_Connect_Expression.SortOrder] = [],
+ _ frame: Spark_Connect_Expression.Window.WindowFrame? = nil
+ ) {
+ self.partitionSpec = partitionSpec
+ self.orderSpec = orderSpec
+ self.frame = frame
+ }
+
+ /// Defines the partitioning columns in a ``WindowSpec``.
+ /// - Parameter colNames: Column names to partition by.
+ /// - Returns: A ``WindowSpec``.
+ public func partitionBy(_ colNames: String...) -> WindowSpec {
+ return partitionBy(colNames.map { Column($0) })
+ }
+
+ /// Defines the partitioning columns in a ``WindowSpec``.
+ /// - Parameter cols: ``Column``s to partition by.
+ /// - Returns: A ``WindowSpec``.
+ public func partitionBy(_ cols: Column...) -> WindowSpec {
+ return partitionBy(cols)
+ }
+
+ func partitionBy(_ cols: [Column]) -> WindowSpec {
+ return WindowSpec(cols.map { $0.expr }, orderSpec, frame)
+ }
+
+ /// Defines the ordering columns in a ``WindowSpec``.
+ /// - Parameter colNames: Column names to order by.
+ /// - Returns: A ``WindowSpec``.
+ public func orderBy(_ colNames: String...) -> WindowSpec {
+ return orderBy(colNames.map { Column($0) })
+ }
+
+ /// Defines the ordering columns in a ``WindowSpec``.
+ /// - Parameter cols: ``Column``s or sort expressions like `col("a").desc()`
to order by.
+ /// - Returns: A ``WindowSpec``.
+ public func orderBy(_ cols: Column...) -> WindowSpec {
+ return orderBy(cols)
+ }
+
+ func orderBy(_ cols: [Column]) -> WindowSpec {
+ return WindowSpec(partitionSpec, cols.map { toSortOrder($0) }, frame)
+ }
+
+ /// Defines the frame boundaries, from `start` (inclusive) to `end`
(inclusive).
+ ///
+ /// Both `start` and `end` are positions relative to the current row. For
example, "0" means
+ /// "current row", while "-1" means the row before the current row, and "5"
means the fifth row
+ /// after the current row.
+ ///
+ /// We recommend users use ``Window/unboundedPreceding``,
``Window/unboundedFollowing``, and
+ /// ``Window/currentRow`` to specify special boundary values, rather than
using integral values
+ /// directly.
+ /// - Parameters:
+ /// - start: A boundary start, inclusive. The frame is unbounded if this is
+ /// ``Window/unboundedPreceding``.
+ /// - end: A boundary end, inclusive. The frame is unbounded if this is
+ /// ``Window/unboundedFollowing``.
+ /// - Returns: A ``WindowSpec``.
+ public func rowsBetween(_ start: Int64, _ end: Int64) throws -> WindowSpec {
+ return withFrame(
+ .row, try rowBoundary(start, Window.unboundedPreceding),
+ try rowBoundary(end, Window.unboundedFollowing))
+ }
+
+ /// Defines the frame boundaries, from `start` (inclusive) to `end`
(inclusive).
+ ///
+ /// Both `start` and `end` are relative to the current row based on the
actual value of the
+ /// ORDER BY expression(s). For example, "0" means "current row", while "-1"
means one off
+ /// before the current row, and "5" means the five off after the current row.
+ ///
+ /// We recommend users use ``Window/unboundedPreceding``,
``Window/unboundedFollowing``, and
+ /// ``Window/currentRow`` to specify special boundary values, rather than
using integral values
+ /// directly.
+ /// - Parameters:
+ /// - start: A boundary start, inclusive. The frame is unbounded if this is
+ /// ``Window/unboundedPreceding``.
+ /// - end: A boundary end, inclusive. The frame is unbounded if this is
+ /// ``Window/unboundedFollowing``.
+ /// - Returns: A ``WindowSpec``.
+ public func rangeBetween(_ start: Int64, _ end: Int64) -> WindowSpec {
+ return withFrame(
+ .range, rangeBoundary(start, Window.unboundedPreceding),
+ rangeBoundary(end, Window.unboundedFollowing))
+ }
+
+ private func withFrame(
+ _ frameType: Spark_Connect_Expression.Window.WindowFrame.FrameType,
+ _ lower: Spark_Connect_Expression.Window.WindowFrame.FrameBoundary,
+ _ upper: Spark_Connect_Expression.Window.WindowFrame.FrameBoundary
+ ) -> WindowSpec {
+ var frame = Spark_Connect_Expression.Window.WindowFrame()
+ frame.frameType = frameType
+ frame.lower = lower
+ frame.upper = upper
+ return WindowSpec(partitionSpec, orderSpec, frame)
+ }
+
+ /// A row-based boundary is an offset of the row position, so it must fit in
`Int32` like Scala.
+ private func rowBoundary(
+ _ value: Int64, _ unbounded: Int64
+ ) throws -> Spark_Connect_Expression.Window.WindowFrame.FrameBoundary {
+ var boundary = Spark_Connect_Expression.Window.WindowFrame.FrameBoundary()
+ switch value {
+ case Window.currentRow:
+ boundary.currentRow = true
+ case unbounded:
+ boundary.unbounded = true
+ case Int64(Int32.min)...Int64(Int32.max):
+ boundary.value = lit(Int32(value)).expr
+ default:
+ throw SparkConnectError.InvalidArgument
+ }
+ return boundary
+ }
+
+ private func rangeBoundary(
+ _ value: Int64, _ unbounded: Int64
+ ) -> Spark_Connect_Expression.Window.WindowFrame.FrameBoundary {
+ var boundary = Spark_Connect_Expression.Window.WindowFrame.FrameBoundary()
+ switch value {
+ case Window.currentRow:
+ boundary.currentRow = true
+ case unbounded:
+ boundary.unbounded = true
+ default:
+ boundary.value = lit(value).expr
+ }
+ return boundary
+ }
+
+ private func toSortOrder(_ col: Column) ->
Spark_Connect_Expression.SortOrder {
+ if case .sortOrder(let sortOrder) = col.expr.exprType {
+ return sortOrder
+ }
+ var sortOrder = Spark_Connect_Expression.SortOrder()
+ sortOrder.child = col.expr
+ sortOrder.direction = .ascending
+ sortOrder.nullOrdering = .sortNullsFirst
+ return sortOrder
+ }
+}
diff --git a/Tests/SparkConnectTests/WindowTests.swift
b/Tests/SparkConnectTests/WindowTests.swift
new file mode 100644
index 0000000..6dc9683
--- /dev/null
+++ b/Tests/SparkConnectTests/WindowTests.swift
@@ -0,0 +1,179 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import Testing
+
+@testable import SparkConnect
+
+/// A test suite for `Window`, `WindowSpec`, and window functions
+@Suite(.serialized)
+struct WindowTests {
+
+ @Test
+ func windowFunctions() throws {
+ for (column, name) in [
+ (cume_dist(), "cume_dist"),
+ (dense_rank(), "dense_rank"),
+ (percent_rank(), "percent_rank"),
+ (rank(), "rank"),
+ (row_number(), "row_number"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.isEmpty)
+ }
+
+ #expect(ntile(4).expr.unresolvedFunction.functionName == "ntile")
+ #expect(ntile(4).expr.unresolvedFunction.arguments[0].literal.integer == 4)
+
+ for (column, name, count) in [
+ (lag(col("a"), 1), "lag", 2),
+ (lag(col("a"), 1, 0), "lag", 3),
+ (lead(col("a"), 1), "lead", 2),
+ (lead(col("a"), 1, 0), "lead", 3),
+ (nth_value(col("a"), 2), "nth_value", 2),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == count)
+
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ #expect(expr.unresolvedFunction.arguments[1].literal.integer != 0)
+ }
+ }
+
+ @Test
+ func over() throws {
+ let window = Window.partitionBy("dept").orderBy(col("salary").desc())
+ let expr = rank().over(window).expr
+ #expect(expr.window.windowFunction.unresolvedFunction.functionName ==
"rank")
+ #expect(expr.window.partitionSpec.count == 1)
+
#expect(expr.window.partitionSpec[0].unresolvedAttribute.unparsedIdentifier ==
"dept")
+ #expect(expr.window.orderSpec.count == 1)
+
#expect(expr.window.orderSpec[0].child.unresolvedAttribute.unparsedIdentifier
== "salary")
+ #expect(expr.window.orderSpec[0].direction == .descending)
+ #expect(!expr.window.hasFrameSpec)
+
+ let ascending = Window.orderBy("a", "b")
+ let ascendingExpr = row_number().over(ascending).expr
+ #expect(ascendingExpr.window.partitionSpec.isEmpty)
+ #expect(ascendingExpr.window.orderSpec.count == 2)
+ #expect(ascendingExpr.window.orderSpec[0].direction == .ascending)
+
+ let emptyExpr = count(col("a")).over().expr
+ #expect(emptyExpr.window.windowFunction.unresolvedFunction.functionName ==
"count")
+ #expect(emptyExpr.window.partitionSpec.isEmpty)
+ #expect(emptyExpr.window.orderSpec.isEmpty)
+ #expect(!emptyExpr.window.hasFrameSpec)
+ }
+
+ @Test
+ func rowsBetween() throws {
+ let spec = try Window.partitionBy(col("a")).orderBy(col("b"))
+ .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+ let frame = try #require(spec.frame)
+ #expect(frame.frameType == .row)
+ #expect(frame.lower.unbounded)
+ #expect(frame.upper.currentRow)
+
+ let bounded = try #require(try Window.orderBy("a").rowsBetween(-1,
2).frame)
+ #expect(bounded.frameType == .row)
+ #expect(bounded.lower.value.literal.integer == -1)
+ #expect(bounded.upper.value.literal.integer == 2)
+
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try Window.orderBy("a").rowsBetween(Int64(Int32.min) - 1,
Window.currentRow)
+ }
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try Window.orderBy("a").rowsBetween(Window.currentRow, Int64(Int32.max)
+ 1)
+ }
+ }
+
+ @Test
+ func rangeBetween() throws {
+ let spec = Window.orderBy("a").rangeBetween(Window.unboundedPreceding,
Window.unboundedFollowing)
+ let frame = try #require(spec.frame)
+ #expect(frame.frameType == .range)
+ #expect(frame.lower.unbounded)
+ #expect(frame.upper.unbounded)
+
+ let bounded = try
#require(Window.orderBy("a").rangeBetween(Window.currentRow, 10000000000).frame)
+ #expect(bounded.frameType == .range)
+ #expect(bounded.lower.currentRow)
+ #expect(bounded.upper.value.literal.long == 10000000000)
+ }
+
+ @Test
+ func rankOverWindow() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES ('a', 100), ('a', 200), ('a', 200), ('b', 300) AS
t(dept, salary)")
+ let window = Window.partitionBy("dept").orderBy(col("salary").desc())
+ let rows = try await df
+ .withColumn("rank", rank().over(window))
+ .withColumn("dense_rank", dense_rank().over(window))
+ .withColumn("row_number",
row_number().over(Window.partitionBy("dept").orderBy("salary")))
+ .orderBy(col("dept"), col("salary"), col("row_number"))
+ .collect()
+ #expect(
+ rows == [
+ Row("a", 100, 3, 2, 1),
+ Row("a", 200, 1, 1, 2),
+ Row("a", 200, 1, 1, 3),
+ Row("b", 300, 1, 1, 1),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func sumOverRowsBetween() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES ('a', 1), ('a', 2), ('a', 3), ('b', 10) AS t(dept,
v)")
+ let window = try Window.partitionBy("dept").orderBy("v")
+ .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+ let rows = try await df
+ .withColumn("sum", sum(col("v")).over(window))
+ .orderBy(col("dept"), col("v"))
+ .collect()
+ #expect(rows == [Row("a", 1, 1), Row("a", 2, 3), Row("a", 3, 6), Row("b",
10, 10)])
+ await spark.stop()
+ }
+
+ @Test
+ func lagLeadOverWindow() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES ('a', 1), ('a', 2), ('a', 3), ('b', 10) AS t(dept,
v)")
+ let window = Window.partitionBy("dept").orderBy("v")
+ let rows = try await df
+ .withColumn("lag", lag(col("v"), 1).over(window))
+ .withColumn("lead", lead(col("v"), 1, -1).over(window))
+ .withColumn("nth", nth_value(col("v"), 2).over(window))
+ .orderBy(col("dept"), col("v"))
+ .collect()
+ #expect(
+ rows == [
+ Row("a", 1, nil, 2, nil),
+ Row("a", 2, 1, 3, 2),
+ Row("a", 3, 2, -1, 2),
+ Row("b", 10, nil, -1, nil),
+ ])
+ await spark.stop()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]