This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 8263408 [SPARK-58433] Support `repartitionByRange` for `DataFrame`
8263408 is described below
commit 826340836f3d79c537a29ed7454e8f77da5c5416
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 14:25:01 2026 -0700
[SPARK-58433] Support `repartitionByRange` for `DataFrame`
### What changes were proposed in this pull request?
This PR aims to support `repartitionByRange` for `DataFrame`.
### Why are the changes needed?
To improve the feature parity with other language clients.
```swift
let df = try await spark.range(2025)
try await df.repartitionByRange(3, "id").write.orc(path)
```
### Does this PR introduce _any_ user-facing change?
No, this is a new API addition.
### How was this patch tested?
Pass the CIs with the newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #453 from dongjoon-hyun/SPARK-58433.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../SparkConnect/DataFrame+Transformations.swift | 28 ++++++++++++++++++++++
Sources/SparkConnect/SparkConnectClient.swift | 19 +++++++++++++++
Tests/SparkConnectTests/DataFrameTests.swift | 16 +++++++++++++
3 files changed, 63 insertions(+)
diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift
b/Sources/SparkConnect/DataFrame+Transformations.swift
index 8dc9db5..ecdbe8e 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -740,6 +740,34 @@ extension DataFrame {
numPartitions: numPartitions, partitionExprs: partitionExprs)
}
+ private func buildRepartitionByRange(numPartitions: Int32?, partitionExprs:
[String])
+ -> DataFrame
+ {
+ let plan = SparkConnectClient.getRepartitionByRange(
+ self.plan.root, partitionExprs, numPartitions)
+ return DataFrame(spark: self.spark, plan: plan)
+ }
+
+ /// Returns a new ``DataFrame`` partitioned by the given partitioning
expressions, using
+ /// `spark.sql.shuffle.partitions` as number of partitions. The resulting
Dataset is range
+ /// partitioned.
+ /// - Parameter partitionExprs: The partition expression strings.
+ /// - Returns: A `DataFrame`.
+ public func repartitionByRange(_ partitionExprs: String...) -> DataFrame {
+ return buildRepartitionByRange(numPartitions: nil, partitionExprs:
partitionExprs)
+ }
+
+ /// Returns a new ``DataFrame`` partitioned by the given partitioning
expressions into
+ /// `numPartitions`. The resulting Dataset is range partitioned.
+ /// - Parameters:
+ /// - numPartitions: The number of partitions.
+ /// - partitionExprs: The partition expression strings.
+ /// - Returns: A `DataFrame`.
+ public func repartitionByRange(_ numPartitions: Int32, _ partitionExprs:
String...) -> DataFrame {
+ return buildRepartitionByRange(
+ numPartitions: numPartitions, partitionExprs: partitionExprs)
+ }
+
/// Returns a new ``DataFrame`` that has exactly `numPartitions` partitions,
when the fewer partitions
/// are requested. If a larger number of partitions is requested, it will
stay at the current
/// number of partitions. Similar to coalesce defined on an `RDD`, this
operation results in a
diff --git a/Sources/SparkConnect/SparkConnectClient.swift
b/Sources/SparkConnect/SparkConnectClient.swift
index f4a9be9..2f3d7d7 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -1142,6 +1142,25 @@ public actor SparkConnectClient {
return createPlan { $0.repartitionByExpression = repartitionByExpression }
}
+ static func getRepartitionByRange(
+ _ child: Relation, _ partitionExprs: [String], _ numPartitions: Int32? =
nil
+ ) -> Plan {
+ var repartitionByExpression = RepartitionByExpression()
+ repartitionByExpression.input = child
+ repartitionByExpression.partitionExprs = partitionExprs.map {
+ var sortOrder = Spark_Connect_Expression.SortOrder()
+ sortOrder.child.exprType = .unresolvedAttribute($0.toUnresolvedAttribute)
+ sortOrder.direction = .ascending
+ var expression = Spark_Connect_Expression()
+ expression.exprType = .sortOrder(sortOrder)
+ return expression
+ }
+ if let numPartitions {
+ repartitionByExpression.numPartitions = numPartitions
+ }
+ return createPlan { $0.repartitionByExpression = repartitionByExpression }
+ }
+
static func getUnpivot(
_ child: Relation,
_ ids: [String],
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift
b/Tests/SparkConnectTests/DataFrameTests.swift
index 231d4ee..f748d3a 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -840,6 +840,22 @@ struct DataFrameTests {
await spark.stop()
}
+ @Test
+ func repartitionByRange() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let tmpDir = "/tmp/" + UUID().uuidString
+ let df = try await spark.range(2025)
+ for n in [1, 3, 5] as [Int32] {
+ try await df.repartitionByRange(n,
"id").write.mode("overwrite").orc(tmpDir)
+ #expect(try await spark.read.orc(tmpDir).inputFiles().count == n)
+ }
+ try await spark.range(1).repartitionByRange(10,
"id").write.mode("overwrite").orc(tmpDir)
+ #expect(try await spark.read.orc(tmpDir).inputFiles().count < 10)
+ try await
spark.range(1).repartitionByRange("id").write.mode("overwrite").orc(tmpDir)
+ #expect(try await spark.read.orc(tmpDir).inputFiles().count < 10)
+ await spark.stop()
+ }
+
@Test
func coalesce() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]