This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 348a364  [SPARK-61062][CONNECT] Upload large local relations at the 
cache threshold, whichever path encodes them
348a364 is described below

commit 348a3643f7646382d4dc24509244055b9cb1f78a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:41:22 2026 +0900

    [SPARK-61062][CONNECT] Upload large local relations at the cache threshold, 
whichever path encodes them
    
    ### What changes were proposed in this pull request?
    
    `createDataFrame` reads the server's local-relation settings for a relation 
of at least 32 MiB of Arrow data, whichever path encoded it, and then sends it 
as the Arrow importer does: inline below 
`spark.sql.session.localRelationCacheThreshold`, and uploaded as cache 
artifacts at it, as PySpark does. This covers local rows 
(`SparkSession::create_dataframe`), the pandas row conversion, 
`create_dataframe_from_columns`, and the Arrow IPC that the float64 and string 
pandas fast paths (#207 [...]
    
    ### Why are the changes needed?
    
    [SPARK-61062](https://issues.apache.org/jira/browse/SPARK-61062): these 
paths sent every relation inline, so one beyond the server's gRPC message size 
limit (128 MiB by default) failed. Against a local Spark 4.3.0 server:
    
    | Input | Current master | This change | PySpark 4.3.0 |
    | --- | --- | --- | --- |
    | pandas, 15,000 strings of 10 KB (about 150 MB) | `h2 protocol error` | 
15,000 rows, 2.8 s | 15,000 rows, 2.4 s |
    | pandas, 100,000 rows of 200 `float64` columns (160 MB) | `h2 protocol 
error` | the same sum as PySpark | the same sum |
    | local rows, 80,000 rows of a 2 KB string and a long (170 MB) | `h2 
protocol error` | the same count and sum, 2.7 s | the same, 3.5 s |
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Large local relations are uploaded at the cache threshold instead of 
failing, as in PySpark.
    
    ### How was this patch tested?
    
    - New tests in `python/tests/test_arrow_handoff.py`, against the mock Arrow 
input service: relations of local rows, of the pandas row conversion, and of 
the float64 and string fast paths above 32 MiB read the cache threshold and are 
uploaded at it, with all their rows; a small relation stays inline without 
reading it.
    - The float64 and string fast-path tests (301 passed), and the lib tests of 
`apache-spark-connect` (461 passed).
    - The reproductions above against a local Spark 4.3.0 server.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #253 from HyukjinKwon/SPARK-61062-large-local-relations.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs        |  22 ++++--
 crates/spark-connect/src/arrow_input.rs |   2 +-
 crates/spark-connect/src/session.rs     | 117 +++++++++++++++++++++++---------
 python/tests/test_arrow_handoff.py      |  48 +++++++++++++
 4 files changed, 150 insertions(+), 39 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 8ec0452..7f2cc5d 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1039,9 +1039,14 @@ impl PySparkSession {
                 let schema = DataType::Struct {
                     fields: fields.clone(),
                 };
-                let frame = self
-                    .session
-                    .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, 
schema);
+                let ipc: Vec<u8> = ipc.extract()?;
+                let frame = py
+                    .detach(|| {
+                        self.session
+                            .create_dataframe_from_inline_arrow_ipc(ipc, 
schema)
+                            .map_err(Box::new)
+                    })
+                    .to_pyerr()?;
                 // Match pandas row input: explicit nonempty field names travel
                 // through toDF as well as the local relation's logical schema.
                 return Ok(PyDataFrame::new(
@@ -1080,9 +1085,14 @@ impl PySparkSession {
                 let schema = DataType::Struct {
                     fields: fields.clone(),
                 };
-                let frame = self
-                    .session
-                    .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, 
schema);
+                let ipc: Vec<u8> = ipc.extract()?;
+                let frame = py
+                    .detach(|| {
+                        self.session
+                            .create_dataframe_from_inline_arrow_ipc(ipc, 
schema)
+                            .map_err(Box::new)
+                    })
+                    .to_pyerr()?;
                 // Match pandas row input: explicit nonempty field names travel
                 // through toDF as well as the local relation's logical schema.
                 return Ok(PyDataFrame::new(
diff --git a/crates/spark-connect/src/arrow_input.rs 
b/crates/spark-connect/src/arrow_input.rs
index 1592468..7d707b9 100644
--- a/crates/spark-connect/src/arrow_input.rs
+++ b/crates/spark-connect/src/arrow_input.rs
@@ -120,7 +120,7 @@ fn validate(batches: &[RecordBatch]) -> Result<()> {
     Ok(())
 }
 
-fn batch_data_size(batch: &RecordBatch) -> usize {
+pub(crate) fn batch_data_size(batch: &RecordBatch) -> usize {
     batch
         .columns()
         .iter()
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 5b6698e..6b0844a 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -418,16 +418,42 @@ impl SparkSession {
         // Empty data -> send the schema only (no Arrow `data`); the server 
builds an empty
         // relation from the schema. A zero-row Arrow payload would be empty 
bytes, which the
         // server cannot parse ("Unexpected end of input. Missing schema.").
-        let data = if rows.is_empty() {
-            None
-        } else {
-            Some(rows_to_arrow_ipc(&rows, &schema)?)
-        };
-        let plan = LogicalPlan::LocalRelation {
+        if rows.is_empty() {
+            let plan = LogicalPlan::LocalRelation {
+                schema: Some(schema),
+                data: None,
+            };
+            return Ok(DataFrame::new(self.clone(), plan));
+        }
+        let batch = rows_to_arrow_batch(&rows, &schema)?;
+        Ok(DataFrame::new(
+            self.clone(),
+            self.local_relation(batch, schema)?,
+        ))
+    }
+
+    /// The local relation of `batch`: inline, without reading the server's 
configuration, when
+    /// it is smaller than [`LOCAL_RELATION_SETTINGS_BYTES`]; otherwise as the 
Arrow importer
+    /// sends one, which reads the server's local-relation settings and 
uploads the data as
+    /// cache artifacts at `spark.sql.session.localRelationCacheThreshold`, as 
PySpark does. A
+    /// single gRPC message cannot carry a relation beyond the server's 
message size limit.
+    fn local_relation(
+        &self,
+        batch: arrow::record_batch::RecordBatch,
+        schema: DataType,
+    ) -> Result<LogicalPlan> {
+        if crate::arrow_input::batch_data_size(&batch) >= 
LOCAL_RELATION_SETTINGS_BYTES {
+            return crate::arrow_input::create_relation(
+                &self.client,
+                vec![batch],
+                Some(schema),
+                None,
+            );
+        }
+        Ok(LogicalPlan::LocalRelation {
             schema: Some(schema),
-            data,
-        };
-        Ok(DataFrame::new(self.clone(), plan))
+            data: Some(batch_to_arrow_ipc(&batch)?),
+        })
     }
 
     /// Create an inline local relation from columns with the same encoding as 
local rows.
@@ -441,29 +467,39 @@ impl SparkSession {
     ) -> Result<DataFrame> {
         let arrow_schema = 
arrow::datatypes::Schema::new(schema_to_arrow_fields(&schema)?);
         let row_count = columns.first().map_or(0, |column| column.len());
-        let data = Some(columns_to_arrow_ipc(columns, arrow_schema, 
row_count)?);
-        let plan = LogicalPlan::LocalRelation {
-            schema: Some(schema),
-            data,
-        };
-        Ok(DataFrame::new(self.clone(), plan))
+        let batch = columns_to_arrow_batch(columns, arrow_schema, row_count)?;
+        Ok(DataFrame::new(
+            self.clone(),
+            self.local_relation(batch, schema)?,
+        ))
     }
 
     /// Internal binding bridge for Arrow IPC already validated and encoded 
from
-    /// local rows or columns. Retains the row path's inline transport policy.
+    /// local rows or columns. Retains the row path's transport policy: 
inline, unless the
+    /// IPC is large enough to consult the server's local-relation settings.
     #[doc(hidden)]
     pub fn create_dataframe_from_inline_arrow_ipc(
         &self,
         data: Vec<u8>,
         schema: DataType,
-    ) -> DataFrame {
-        DataFrame::new(
-            self.clone(),
+    ) -> Result<DataFrame> {
+        let plan = if data.len() >= LOCAL_RELATION_SETTINGS_BYTES {
+            let reader =
+                
arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(data), None)
+                    .map_err(|e| {
+                        SparkError::connect_msg(format!("Failed to read Arrow 
IPC: {e}"))
+                    })?;
+            let batches = reader
+                .collect::<std::result::Result<Vec<_>, _>>()
+                .map_err(|e| SparkError::connect_msg(format!("Failed to read 
Arrow IPC: {e}")))?;
+            crate::arrow_input::create_relation(&self.client, batches, 
Some(schema), None)?
+        } else {
             LogicalPlan::LocalRelation {
                 schema: Some(schema),
                 data: Some(data),
-            },
-        )
+            }
+        };
+        Ok(DataFrame::new(self.clone(), plan))
     }
 
     /// Create a local DataFrame from Arrow batches without materializing rows.
@@ -895,13 +931,18 @@ fn row_width_mismatch(row_index: usize, actual: usize, 
expected: usize) -> Spark
 
 /// Convert Arrow RecordBatch data to IPC bytes.
 /// Used by createDataFrame to package local data.
-fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
+/// The size of a local relation, in Arrow data, from which `createDataFrame` 
reads the
+/// server's local-relation settings to decide whether to upload it: half of 
the default
+/// `spark.sql.session.localRelationCacheThreshold`, and well below the 
default gRPC message
+/// size limit of the server (128 MiB). Smaller relations are sent inline 
without that RPC.
+const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20;
+
+fn rows_to_arrow_batch(
+    rows: &[Row],
+    schema: &DataType,
+) -> Result<arrow::record_batch::RecordBatch> {
     use arrow::datatypes::Schema as ArrowSchema;
 
-    if rows.is_empty() {
-        return Ok(vec![]);
-    }
-
     // Convert our schema to Arrow schema
     let arrow_schema_vec = schema_to_arrow_fields(schema)?;
     let arrow_schema = ArrowSchema::new(arrow_schema_vec);
@@ -925,21 +966,33 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> 
Result<Vec<u8>> {
         columns.push(values_to_arrow(&column, &field.data_type)?);
     }
 
-    columns_to_arrow_ipc(columns, arrow_schema, rows.len())
+    columns_to_arrow_batch(columns, arrow_schema, rows.len())
+}
+
+/// The Arrow IPC stream of local rows, as the server reads a local relation.
+#[cfg(test)]
+fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
+    if rows.is_empty() {
+        return Ok(vec![]);
+    }
+    batch_to_arrow_ipc(&rows_to_arrow_batch(rows, schema)?)
 }
 
-fn columns_to_arrow_ipc(
+fn columns_to_arrow_batch(
     columns: Vec<arrow::array::ArrayRef>,
     arrow_schema: arrow::datatypes::Schema,
     row_count: usize,
-) -> Result<Vec<u8>> {
-    use arrow::ipc::writer::StreamWriter;
+) -> Result<arrow::record_batch::RecordBatch> {
     use arrow::record_batch::RecordBatch;
 
     // Preserve the row count even for a batch without columns.
     let options = 
arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(row_count));
-    let batch = RecordBatch::try_new_with_options(Arc::new(arrow_schema), 
columns, &options)
-        .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow 
batch: {}", e)))?;
+    RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, 
&options)
+        .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow 
batch: {}", e)))
+}
+
+fn batch_to_arrow_ipc(batch: &arrow::record_batch::RecordBatch) -> 
Result<Vec<u8>> {
+    use arrow::ipc::writer::StreamWriter;
 
     // Write to IPC stream
     let mut buffer = Vec::new();
diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index 738c3fb..0fd5a5b 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -1177,6 +1177,54 @@ def 
test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path):
         }
 
 
+# Local relations of at least 32 MiB read the server's local-relation 
settings, and are uploaded
+# as cache artifacts at its threshold, whichever path encoded them; a single 
gRPC message
+# cannot carry a relation beyond the server's message size limit.
+
[email protected]("path", ["local_rows", "pandas_rows", "float64", 
"strings"])
+def test_large_local_relations_follow_the_cache_threshold(protocol, path):
+    # (The mock service keeps gRPC's default 4 MiB message limit, so the 
inline variant of
+    # these inputs, below a higher threshold, can only be checked against a 
real server.)
+    import numpy as np
+
+    if path == "local_rows":
+        data = [("y" * 2000, i) for i in range(20_000)]
+        schema = T.StructType([T.StructField("v", T.StringType()), 
T.StructField("i", T.LongType())])
+        expected_rows = 20_000
+    elif path == "pandas_rows":
+        # float32 columns into DoubleType keep the row path.
+        data = pd.DataFrame(np.ones((100_000, 45), dtype=np.float32),
+                            columns=[f"c{i}" for i in range(45)])
+        schema = T.StructType([T.StructField(c, T.DoubleType()) for c in 
data.columns])
+        expected_rows = 100_000
+    elif path == "float64":
+        data = pd.DataFrame(np.ones((100_000, 45)), columns=[f"c{i}" for i in 
range(45)])
+        schema = T.StructType([T.StructField(c, T.DoubleType()) for c in 
data.columns])
+        expected_rows = 100_000
+    else:
+        data = pd.DataFrame({"v": ["x" * 3500] * 10_000}, dtype=object)
+        schema = T.StructType([T.StructField("v", T.StringType())])
+        expected_rows = 10_000
+    overrides = {"spark.sql.session.localRelationCacheThreshold": "1"}
+    with _arrow_input_service(protocol, **overrides) as (spark, state):
+        df = spark.createDataFrame(data, schema)
+        df.toArrow()
+        assert sum(table.num_rows for table in state["input_tables"]) == 
expected_rows
+        assert state["uploads"]
+        assert any("spark.sql.session.localRelationCacheThreshold" in keys
+                   for _, keys in state["config_requests"])
+
+
+def test_small_local_relations_stay_inline_without_reading_settings(protocol):
+    with _arrow_input_service(protocol) as (spark, state):
+        schema = T.StructType([T.StructField("v", T.StringType()), 
T.StructField("i", T.LongType())])
+        spark.createDataFrame([("y" * 100, i) for i in range(100)], 
schema).toArrow()
+        assert not state["uploads"]
+        assert not any("spark.sql.session.localRelationCacheThreshold" in keys
+                       for _, keys in state["config_requests"])
+        assert state["input_tables"][0].num_rows == 100
+
+
 # Signed-int64 pandas input uses the same Arrow importer and mock service.
 
 @pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", 
"python_nan", "pyarrow_nan"])


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to