This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 8263408  [SPARK-58433] Support `repartitionByRange` for `DataFrame`
8263408 is described below

commit 826340836f3d79c537a29ed7454e8f77da5c5416
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 14:25:01 2026 -0700

    [SPARK-58433] Support `repartitionByRange` for `DataFrame`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `repartitionByRange` for `DataFrame`.
    
    ### Why are the changes needed?
    
    To improve the feature parity with other language clients.
    
    ```swift
    let df = try await spark.range(2025)
    try await df.repartitionByRange(3, "id").write.orc(path)
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this is a new API addition.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #453 from dongjoon-hyun/SPARK-58433.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../SparkConnect/DataFrame+Transformations.swift   | 28 ++++++++++++++++++++++
 Sources/SparkConnect/SparkConnectClient.swift      | 19 +++++++++++++++
 Tests/SparkConnectTests/DataFrameTests.swift       | 16 +++++++++++++
 3 files changed, 63 insertions(+)

diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift 
b/Sources/SparkConnect/DataFrame+Transformations.swift
index 8dc9db5..ecdbe8e 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -740,6 +740,34 @@ extension DataFrame {
       numPartitions: numPartitions, partitionExprs: partitionExprs)
   }
 
+  private func buildRepartitionByRange(numPartitions: Int32?, partitionExprs: 
[String])
+    -> DataFrame
+  {
+    let plan = SparkConnectClient.getRepartitionByRange(
+      self.plan.root, partitionExprs, numPartitions)
+    return DataFrame(spark: self.spark, plan: plan)
+  }
+
+  /// Returns a new ``DataFrame`` partitioned by the given partitioning 
expressions, using
+  /// `spark.sql.shuffle.partitions` as number of partitions. The resulting 
Dataset is range
+  /// partitioned.
+  /// - Parameter partitionExprs: The partition expression strings.
+  /// - Returns: A `DataFrame`.
+  public func repartitionByRange(_ partitionExprs: String...) -> DataFrame {
+    return buildRepartitionByRange(numPartitions: nil, partitionExprs: 
partitionExprs)
+  }
+
+  /// Returns a new ``DataFrame`` partitioned by the given partitioning 
expressions into
+  /// `numPartitions`. The resulting Dataset is range partitioned.
+  /// - Parameters:
+  ///   - numPartitions: The number of partitions.
+  ///   - partitionExprs: The partition expression strings.
+  /// - Returns: A `DataFrame`.
+  public func repartitionByRange(_ numPartitions: Int32, _ partitionExprs: 
String...) -> DataFrame {
+    return buildRepartitionByRange(
+      numPartitions: numPartitions, partitionExprs: partitionExprs)
+  }
+
   /// Returns a new ``DataFrame`` that has exactly `numPartitions` partitions, 
when the fewer partitions
   /// are requested. If a larger number of partitions is requested, it will 
stay at the current
   /// number of partitions. Similar to coalesce defined on an `RDD`, this 
operation results in a
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index f4a9be9..2f3d7d7 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -1142,6 +1142,25 @@ public actor SparkConnectClient {
     return createPlan { $0.repartitionByExpression = repartitionByExpression }
   }
 
+  static func getRepartitionByRange(
+    _ child: Relation, _ partitionExprs: [String], _ numPartitions: Int32? = 
nil
+  ) -> Plan {
+    var repartitionByExpression = RepartitionByExpression()
+    repartitionByExpression.input = child
+    repartitionByExpression.partitionExprs = partitionExprs.map {
+      var sortOrder = Spark_Connect_Expression.SortOrder()
+      sortOrder.child.exprType = .unresolvedAttribute($0.toUnresolvedAttribute)
+      sortOrder.direction = .ascending
+      var expression = Spark_Connect_Expression()
+      expression.exprType = .sortOrder(sortOrder)
+      return expression
+    }
+    if let numPartitions {
+      repartitionByExpression.numPartitions = numPartitions
+    }
+    return createPlan { $0.repartitionByExpression = repartitionByExpression }
+  }
+
   static func getUnpivot(
     _ child: Relation,
     _ ids: [String],
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index 231d4ee..f748d3a 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -840,6 +840,22 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func repartitionByRange() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let tmpDir = "/tmp/" + UUID().uuidString
+    let df = try await spark.range(2025)
+    for n in [1, 3, 5] as [Int32] {
+      try await df.repartitionByRange(n, 
"id").write.mode("overwrite").orc(tmpDir)
+      #expect(try await spark.read.orc(tmpDir).inputFiles().count == n)
+    }
+    try await spark.range(1).repartitionByRange(10, 
"id").write.mode("overwrite").orc(tmpDir)
+    #expect(try await spark.read.orc(tmpDir).inputFiles().count < 10)
+    try await 
spark.range(1).repartitionByRange("id").write.mode("overwrite").orc(tmpDir)
+    #expect(try await spark.read.orc(tmpDir).inputFiles().count < 10)
+    await spark.stop()
+  }
+
   @Test
   func coalesce() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to