This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new fa11a94  [SPARK-58432] Support `sortWithinPartitions` for `DataFrame`
fa11a94 is described below

commit fa11a944e1759bcdc57ec030ace6f3bff0906bdb
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 14:23:48 2026 -0700

    [SPARK-58432] Support `sortWithinPartitions` for `DataFrame`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `sortWithinPartitions` API in `DataFrame`.
    
    - `sortWithinPartitions(_ cols: String...)`
    
    This is the same operation as `SORT BY` in SQL (Hive QL). It reuses the 
existing `Sort`
    relation with `is_global = false`.
    
    ### Why are the changes needed?
    
    For feature parity with the other Spark Connect clients (Scala/Python) 
which provide
    `sortWithinPartitions`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a new API.
    
    ### How was this patch tested?
    
    Pass the CIs.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #452 from dongjoon-hyun/SPARK-58432.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/DataFrame+Transformations.swift |  9 +++++++++
 Sources/SparkConnect/SparkConnectClient.swift        |  4 ++--
 Tests/SparkConnectTests/DataFrameTests.swift         | 10 ++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift 
b/Sources/SparkConnect/DataFrame+Transformations.swift
index cc4c0f6..8dc9db5 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -260,6 +260,15 @@ extension DataFrame {
     return DataFrame(spark: self.spark, plan: 
SparkConnectClient.getSort(self.plan.root, cols))
   }
 
+  /// Returns a new ``DataFrame`` with each partition sorted by the specified 
column(s).
+  /// This is the same operation as `SORT BY` in SQL (Hive QL).
+  /// - Parameter cols: Column names.
+  /// - Returns: A ``DataFrame`` with each partition sorted.
+  public func sortWithinPartitions(_ cols: String...) -> DataFrame {
+    return DataFrame(
+      spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root, 
cols, false))
+  }
+
   /// Limits the result count to the number specified.
   ///
   /// This transformation is often used for:
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index b6b21d3..f4a9be9 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -702,7 +702,7 @@ public actor SparkConnectClient {
     return createPlan { $0.freqItems = freqItems }
   }
 
-  static func getSort(_ child: Relation, _ cols: [String]) -> Plan {
+  static func getSort(_ child: Relation, _ cols: [String], _ isGlobal: Bool = 
true) -> Plan {
     var sort = Sort()
     sort.input = child
     let expressions: [Spark_Connect_Expression.SortOrder] = cols.map {
@@ -712,7 +712,7 @@ public actor SparkConnectClient {
       return expression
     }
     sort.order = expressions
-    sort.isGlobal = true
+    sort.isGlobal = isGlobal
     return createPlan { $0.sort = sort }
   }
 
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index 0962a5d..231d4ee 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -436,6 +436,16 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func sortWithinPartitions() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let expected = Array((1...10).map { Row($0) })
+    #expect(
+      try await spark.range(10, 0, 
-1).repartition(1).sortWithinPartitions("id").collect()
+        == expected)
+    await spark.stop()
+  }
+
   @Test
   func table() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to