This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 40db9c0  [SPARK-58530] Support large local data in `createDataFrame` 
via `CachedLocalRelation`
40db9c0 is described below

commit 40db9c0a19f6c833ace32f0aa75d62c621708eb7
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Aug 3 15:34:57 2026 -0700

    [SPARK-58530] Support large local data in `createDataFrame` via 
`CachedLocalRelation`
    
    ### What changes were proposed in this pull request?
    
    This PR supports large local data in `SparkSession.createDataFrame` via 
`CachedLocalRelation`.
    When the serialized Arrow IPC stream is equal to or larger than
    `spark.sql.session.localRelationCacheThreshold` (1MiB by default), the 
serialized
    `LocalRelation` is uploaded to the server as a `cache/<sha256>` artifact 
and the plan
    references it with a `CachedLocalRelation` message, instead of inlining the 
data into the
    plan. This follows the PySpark client behavior.
    
    `CachedLocalRelation` (supported since Spark 3.5) is used instead of the 
newer
    `ChunkedCachedLocalRelation` (added in Spark 4.1.0 by SPARK-53917) in order 
to support all
    Spark Connect servers this library targets, including 4.0.x.
    
    ### Why are the changes needed?
    
    Previously, `createDataFrame` inlined the data into the plan, so it was 
limited by the gRPC
    message size limit and threw `LocalRelationTooLarge` for larger data. With 
this PR, larger
    data is transparently uploaded via the artifact API (SPARK-58528) and 
re-uploading identical
    data is skipped based on its SHA-256 hash.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `createDataFrame` now succeeds for data whose serialized size exceeds 
the gRPC message
    size limit. The API signature is unchanged.
    
    ### How was this patch tested?
    
    Pass the CIs with a new test case (`CreateDataFrameTests.largeData`) which 
creates a
    DataFrame with 10,000 rows (~2MiB serialized, above the default 1MiB 
threshold), verifies
    `count`/`sum`/`filter` round trips against a live Spark Connect server, and 
verifies that a
    second `createDataFrame` call with the same data skips the upload.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #473 from dongjoon-hyun/SPARK-58530.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/SparkConnectClient+Artifact.swift | 13 +++++++++++++
 Sources/SparkConnect/SparkConnectClient.swift          | 11 +++++++++++
 Sources/SparkConnect/SparkSession.swift                | 13 ++++++++++---
 Tests/SparkConnectTests/CreateDataFrameTests.swift     | 18 ++++++++++++++++++
 4 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/Sources/SparkConnect/SparkConnectClient+Artifact.swift 
b/Sources/SparkConnect/SparkConnectClient+Artifact.swift
index cf500a0..92e114f 100644
--- a/Sources/SparkConnect/SparkConnectClient+Artifact.swift
+++ b/Sources/SparkConnect/SparkConnectClient+Artifact.swift
@@ -147,4 +147,17 @@ extension SparkConnectClient {
     }
     return hash
   }
+
+  /// Cache a `LocalRelation` with the given `Apache Arrow` IPC stream as a 
`cache/` artifact
+  /// in the server-side session.
+  /// - Parameters:
+  ///   - data: An `Apache Arrow` IPC stream.
+  ///   - schema: A DDL-formatted schema string.
+  /// - Returns: A SHA-256 hash of the serialized `LocalRelation` as a 
lowercase hex string.
+  func cacheLocalRelation(_ data: Data, _ schema: String) async throws -> 
String {
+    var localRelation = Spark_Connect_LocalRelation()
+    localRelation.data = data
+    localRelation.schema = schema
+    return try await cacheArtifact(try localRelation.serializedData())
+  }
 }
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index 6195125..7f43579 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -407,6 +407,17 @@ public actor SparkConnectClient {
     return Self.createPlan { $0.localRelation = localRelation }
   }
 
+  /// Create a `Plan` with a `CachedLocalRelation` referring to a 
`LocalRelation` cached in the
+  /// server-side session.
+  /// - Parameter hash: A SHA-256 hash of the serialized `LocalRelation`, 
returned by
+  ///   ``cacheLocalRelation(_:_:)``.
+  /// - Returns: A `Plan` instance.
+  func getCachedLocalRelation(_ hash: String) -> Plan {
+    var cachedLocalRelation = Spark_Connect_CachedLocalRelation()
+    cachedLocalRelation.hash = hash
+    return Self.createPlan { $0.cachedLocalRelation = cachedLocalRelation }
+  }
+
   /// Create a `Plan` instance for `Range` relation.
   /// - Parameters:
   ///   - start: A start of the range.
diff --git a/Sources/SparkConnect/SparkSession.swift 
b/Sources/SparkConnect/SparkSession.swift
index 8ae3027..4b99641 100644
--- a/Sources/SparkConnect/SparkSession.swift
+++ b/Sources/SparkConnect/SparkSession.swift
@@ -122,9 +122,10 @@ public actor SparkSession {
   /// The data is serialized into an `Apache Arrow` IPC stream and embedded 
into the plan as a
   /// `LocalRelation`. The supported schema types are `BOOLEAN`, `TINYINT`, 
`SMALLINT`, `INT`,
   /// `BIGINT`, `FLOAT`, `DOUBLE`, `STRING`, `BINARY`, `DATE`, and 
`TIMESTAMP`. `nil` values are
-  /// mapped to `NULL`. Since the serialized data is limited by the Spark 
Connect server's gRPC
-  /// message size limit (128MiB by default), 
``SparkConnectError/LocalRelationTooLarge`` is
-  /// thrown for larger data.
+  /// mapped to `NULL`. When the serialized data is equal to or larger than
+  /// `spark.sql.session.localRelationCacheThreshold` (1MiB by default), it is 
uploaded to the
+  /// server as a `cache/` artifact and referenced by a `CachedLocalRelation` 
plan instead, so
+  /// it is not limited by the gRPC message size limit.
   ///
   /// - Parameters:
   ///   - data: An array of rows whose values are ordered like the schema 
fields.
@@ -136,6 +137,12 @@ public actor SparkSession {
       throw SparkConnectError.InvalidType
     }
     let ipcStream = try ConvertToArrow.toArrowIPCStream(data, structType)
+    let threshold = Int(
+      try await 
client.getConf("spark.sql.session.localRelationCacheThreshold"))!
+    if ipcStream.count >= threshold {
+      let hash = try await client.cacheLocalRelation(ipcStream, schema)
+      return await DataFrame(spark: self, plan: 
client.getCachedLocalRelation(hash))
+    }
     guard ipcStream.count <= Self.maxLocalRelationSize else {
       throw SparkConnectError.LocalRelationTooLarge
     }
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift 
b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index ef419de..e2dd454 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -139,4 +139,22 @@ struct CreateDataFrameTests {
     #expect(try rows[15].get(1) as? String == value + "15")
     await spark.stop()
   }
+
+  @Test
+  func largeData() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    // Larger than `spark.sql.session.localRelationCacheThreshold` (1MiB by 
default) in order to
+    // exercise the `CachedLocalRelation`-based cache path.
+    let value = String(repeating: "x", count: 200)
+    let data: [[Sendable?]] = (0..<10000).map { [$0, value] }
+    let df = try await spark.createDataFrame(data, "id INT, value STRING")
+    #expect(try await df.count() == 10000)
+    #expect(try await df.selectExpr("sum(id)").collect() == 
[Row(Int64(49_995_000))])
+    #expect(try await df.filter("id = 9999").collect() == [Row(9999, value)])
+
+    // The second `createDataFrame` call with the same data skips the upload.
+    let df2 = try await spark.createDataFrame(data, "id INT, value STRING")
+    #expect(try await df2.count() == 10000)
+    await spark.stop()
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to