This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 405d54c  [SPARK-58427] Support `randomSplit` for `DataFrame`
405d54c is described below

commit 405d54c58f6e5ce4df9a09cc31a3ce86390cfb23
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 29 13:05:59 2026 -0700

    [SPARK-58427] Support `randomSplit` for `DataFrame`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support `DataFrame.randomSplit` API.
    
    - `randomSplit(_ weights: [Double], _ seed: Int64) throws -> [DataFrame]`
    - `randomSplit(_ weights: [Double]) throws -> [DataFrame]` (with a random 
seed)
    
    Like the other Spark Connect clients (PySpark and Scala), it is implemented 
on the
    client side by normalizing the weights into cumulative ranges and creating 
one
    `Sample` relation per weight with the same seed, `deterministic_order = 
true`, and
    `[lowerBound, upperBound)` ranges. `SparkConnectClient.getSample` is 
extended to
    accept `lowerBound` and `deterministicOrder`.
    
    ### Why are the changes needed?
    
    To improve the API coverage by providing feature parity with the other 
Spark clients.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this is a new API addition.
    
    ### How was this patch tested?
    
    Pass the CIs.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #450 from dongjoon-hyun/SPARK-58427.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../SparkConnect/DataFrame+Transformations.swift   | 33 +++++++++++++++++++++-
 Sources/SparkConnect/SparkConnectClient.swift      |  8 ++++--
 Tests/SparkConnectTests/DataFrameTests.swift       | 21 ++++++++++++++
 3 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/Sources/SparkConnect/DataFrame+Transformations.swift 
b/Sources/SparkConnect/DataFrame+Transformations.swift
index b024719..8c6b1b9 100644
--- a/Sources/SparkConnect/DataFrame+Transformations.swift
+++ b/Sources/SparkConnect/DataFrame+Transformations.swift
@@ -297,7 +297,7 @@ extension DataFrame {
   public func sample(_ withReplacement: Bool, _ fraction: Double, _ seed: 
Int64) -> DataFrame {
     return DataFrame(
       spark: self.spark,
-      plan: SparkConnectClient.getSample(self.plan.root, withReplacement, 
fraction, seed))
+      plan: SparkConnectClient.getSample(self.plan.root, withReplacement, 0.0, 
fraction, seed))
   }
 
   /// Returns a new ``Dataset`` by sampling a fraction of rows, using a random 
seed.
@@ -328,6 +328,37 @@ extension DataFrame {
     return sample(false, fraction)
   }
 
+  /// Randomly splits this ``DataFrame`` with the provided weights, using a 
user-supplied seed.
+  /// - Parameters:
+  ///   - weights: Weights for splits. Must be non-empty and positive. They 
will be normalized
+  ///     if they don't sum up to 1.
+  ///   - seed: Seed for sampling.
+  /// - Returns: An array of ``DataFrame``s whose element count is equal to 
the weight count.
+  public func randomSplit(_ weights: [Double], _ seed: Int64) throws -> 
[DataFrame] {
+    guard !weights.isEmpty, weights.allSatisfy({ $0 > 0 }) else {
+      throw SparkConnectError.InvalidArgument
+    }
+    let sum = weights.reduce(0, +)
+    var normalizedCumWeights: [Double] = [0.0]
+    for weight in weights {
+      normalizedCumWeights.append(normalizedCumWeights.last! + weight / sum)
+    }
+    return (0..<weights.count).map { i in
+      DataFrame(
+        spark: self.spark,
+        plan: SparkConnectClient.getSample(
+          self.plan.root, false, normalizedCumWeights[i], 
normalizedCumWeights[i + 1], seed, true))
+    }
+  }
+
+  /// Randomly splits this ``DataFrame`` with the provided weights, using a 
random seed.
+  /// - Parameter weights: Weights for splits. Must be non-empty and positive. 
They will be
+  ///   normalized if they don't sum up to 1.
+  /// - Returns: An array of ``DataFrame``s whose element count is equal to 
the weight count.
+  public func randomSplit(_ weights: [Double]) throws -> [DataFrame] {
+    return try randomSplit(weights, Int64.random(in: Int64.min...Int64.max))
+  }
+
   // MARK: - Join Operations
 
   /// Join with another DataFrame.
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index 7789cae..e4b57fd 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -731,14 +731,16 @@ public actor SparkConnectClient {
   }
 
   static func getSample(
-    _ child: Relation, _ withReplacement: Bool, _ fraction: Double, _ seed: 
Int64
+    _ child: Relation, _ withReplacement: Bool, _ lowerBound: Double, _ 
upperBound: Double,
+    _ seed: Int64, _ deterministicOrder: Bool = false
   ) -> Plan {
     var sample = Sample()
     sample.input = child
     sample.withReplacement = withReplacement
-    sample.lowerBound = 0.0
-    sample.upperBound = fraction
+    sample.lowerBound = lowerBound
+    sample.upperBound = upperBound
     sample.seed = seed
+    sample.deterministicOrder = deterministicOrder
     return createPlan { $0.sample = sample }
   }
 
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift 
b/Tests/SparkConnectTests/DataFrameTests.swift
index 7a6ea90..43e5822 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -372,6 +372,27 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func randomSplit() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.range(1000)
+    let splits = try await df.randomSplit([1.0, 2.0, 3.0], 0)
+    #expect(splits.count == 3)
+    var total: Int64 = 0
+    for split in splits {
+      total += try await split.count()
+    }
+    #expect(total == 1000)
+    for (df1, df2) in zip(splits, try await df.randomSplit([1.0, 2.0, 3.0], 
0)) {
+      #expect(try await df1.count() == df2.count())
+    }
+    await #expect(throws: SparkConnectError.InvalidArgument) { try await 
df.randomSplit([]) }
+    await #expect(throws: SparkConnectError.InvalidArgument) {
+      try await df.randomSplit([1.0, -1.0])
+    }
+    await spark.stop()
+  }
+
   @Test
   func isEmpty() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to