This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new e3d6129e [SPARK-61143][CONNECT] Decode the Arrow batches that the 
server compresses with zstd or lz4 in the Rust client, as PySpark does
e3d6129e is described below

commit e3d6129e51db59931b712f886a47de64224abbc1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 13:05:00 2026 +0900

    [SPARK-61143][CONNECT] Decode the Arrow batches that the server compresses 
with zstd or lz4 in the Rust client, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61143](https://issues.apache.org/jira/browse/SPARK-61143): With 
`spark.sql.execution.arrow.compression.codec` set to `zstd` or `lz4`, the Spark 
Connect server compresses the Arrow batches of the results. PySpark reads them 
with PyArrow.
    
    The Rust client now builds `arrow` with its `ipc_compression` feature, so 
it decodes the batches of either codec. Both clients get this: the Rust client, 
and the Python client, which decodes the results in Rust.
    
    The only change to `Cargo.lock` is the `zstd` dependency of `arrow-ipc`. 
`zstd`, of `polars-arrow`, and `lz4_flex`, of `arrow-ipc`, were already in the 
lock file.
    
    ### Why are the changes needed?
    
    The client failed to decode the compressed batches:
    
    ```
    PySparkRuntimeError: Failed to decode Arrow batch: Invalid argument error: 
zstd IPC decompression requires the zstd feature
    ```
    
    So under that conf, `collect`, `toArrow` and `toPandas` failed, also on the 
results of pandas UDFs and `applyInPandas`. Nine tests of Apache Spark failed 
on this client, the `*_with_compression_codec*` tests of 
`test_parity_arrow.py`, `test_parity_pandas_grouped_map.py` and 
`test_parity_pandas_udf_scalar.py`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The results compressed by the server with `zstd` or `lz4` are decoded, 
as with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new Rust unit test encodes batches with each codec and checks that they 
decode to the same rows and record batches as uncompressed ones.
    - A new live test collects a DataFrame, and converts it to Arrow and 
pandas, under each codec and `none`. It passes with official PySpark 4.3.0, and 
fails without this change.
    - The 10 `compression` tests of Apache Spark's `test_parity_arrow.py`, 
`test_parity_pandas_grouped_map.py` and `test_parity_pandas_udf_scalar.py` pass 
on this client against a local server.
    - The lib tests of both crates (591 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2202 passed, 8 skipped), and the 
live drop-in and Arrow suites (186 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #336 from HyukjinKwon/SPARK-61143-arrow-compression.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 Cargo.lock                            |  1 +
 Cargo.toml                            |  2 +-
 crates/spark-connect/src/dataframe.rs | 45 +++++++++++++++++++++++++++++++++++
 python/tests/test_dropin_e2e.py       | 15 ++++++++++++
 4 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 29a30b61..0e6dd6e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -288,6 +288,7 @@ dependencies = [
  "arrow-select",
  "flatbuffers",
  "lz4_flex",
+ "zstd",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 68f28243..8595732f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -73,7 +73,7 @@ tokio-stream = "0.1"
 futures = "0.3"
 
 # Arrow (result decoding).
-arrow = { version = "58", default-features = false, features = ["ipc", 
"prettyprint"] }
+arrow = { version = "58", default-features = false, features = ["ipc", 
"ipc_compression", "prettyprint"] }
 
 # DataFusion for the optional to_datafusion() conversion.
 # Pinned to 53 for arrow 58 compatibility.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index c9ae717d..bafdebba 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -3048,6 +3048,51 @@ mod nested_decode_tests {
         row
     }
 
+    #[test]
+    fn batches_compressed_with_zstd_or_lz4_decode_as_uncompressed_ones() {
+        // As PyArrow reads them: the server compresses the Arrow batches of 
the results with
+        // the codec of spark.sql.execution.arrow.compression.codec, zstd or 
lz4.
+        use arrow::ipc::writer::IpcWriteOptions;
+        use arrow::ipc::CompressionType;
+
+        let ids: ArrayRef = 
Arc::new(Int64Array::from_iter((0..2000).map(Some)));
+        let names: ArrayRef = Arc::new(StringArray::from_iter(
+            (0..2000).map(|i| (i % 3 != 0).then(|| format!("name {}", i % 7))),
+        ));
+        let plain = encoded_batches(vec![("id", ids.clone()), ("name", 
names.clone())], 2, false);
+        let rows = decode_arrow_batch(&plain).unwrap();
+        assert_eq!(rows.len(), 4000);
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, true),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![ids, 
names]).unwrap();
+        for codec in [CompressionType::ZSTD, CompressionType::LZ4_FRAME] {
+            let options = IpcWriteOptions::default()
+                .try_with_compression(Some(codec))
+                .unwrap();
+            let mut data = vec![];
+            {
+                let mut writer =
+                    StreamWriter::try_new_with_options(&mut data, &schema, 
options).unwrap();
+                writer.write(&batch).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
+            assert!(data.len() < plain.data.len(), "{codec:?} compresses");
+            let compressed = proto::execute_plan_response::ArrowBatch {
+                data: data.into(),
+                ..Default::default()
+            };
+            assert_eq!(decode_arrow_batch(&compressed).unwrap(), rows, 
"{codec:?}");
+            assert_eq!(
+                decode_arrow_record_batches(&compressed).unwrap(),
+                decode_arrow_record_batches(&plain).unwrap(),
+                "{codec:?}"
+            );
+        }
+    }
+
     #[test]
     fn mixed_nullable_slices_decode_across_empty_and_repeated_batches() {
         let columns: Vec<(&str, ArrayRef)> = vec![
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a6a19f2d..481e48a3 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2083,6 +2083,21 @@ def 
test_collected_rows_keep_their_values_as_pyspark(session):
     assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
 
 
+def test_results_compressed_by_the_server_decode_as_pyspark(session):
+    # As PySpark's: the server compresses the Arrow batches of the results 
with the codec of
+    # spark.sql.execution.arrow.compression.codec, zstd or lz4, which the 
client decodes.
+    df = session.range(0, 20000, 1, 2).selectExpr("id", "CAST(id % 7 AS 
STRING) AS s")
+    rows, table, frame = df.collect(), df.toArrow(), df.toPandas()
+    try:
+        for codec in ("zstd", "lz4", "none"):
+            session.conf.set("spark.sql.execution.arrow.compression.codec", 
codec)
+            assert df.collect() == rows, codec
+            assert df.toArrow().equals(table), codec
+            assert df.toPandas().equals(frame), codec
+    finally:
+        session.conf.unset("spark.sql.execution.arrow.compression.codec")
+
+
 def test_transpose_takes_a_column_name_as_pyspark(session):
     # As PySpark's transpose, of a Column or a column name as the index column.
     df = session.createDataFrame([("a", 1, 2), ("b", 3, 4)], "k string, x int, 
y int")


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to