This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a48cfe4  [SPARK-60972][CONNECT] Simplify DataFrameReader plan 
construction
a48cfe4 is described below

commit a48cfe4efd659a2527ed5cce7426bea60a9b1f91
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Nov 6 14:51:08 2026 +0900

    [SPARK-60972][CONNECT] Simplify DataFrameReader plan construction
    
    DataFrameReader repeats data-source plan construction across its six 
file-format helpers and clones fields even though the read methods consume the 
reader.
    
    Route json, csv, parquet, orc, text, and xml through format(...).load(...), 
and move owned format, schema, and options values into load, table, changes, 
and JDBC plans. Public signatures and generated plans are preserved.
    
    JIRA: https://issues.apache.org/jira/browse/SPARK-60972
    
    Validation:
    - 419 Rust library tests and 11 read/write golden tests passed.
    - All integration tests compile, including with wasm-udf; cargo fmt passes.
    - 12 offline reader tests and 2 live-server reader tests passed against the 
built extension.
    - API-name comparison against PySpark 4.3.0 found zero gaps across the 12 
required classes, after the documented transport exclusions.
    - Official Connect parity gate: 87 files, 2638 passed, 589 skipped, zero 
unexpected failures; uses CI's three cluster-only file exclusions.
    - Workspace clippy reports existing errors in unchanged tests and bindings; 
reproduced on unchanged master. No reported clippy error is in the modified 
file.
    
    This pull request and its description were written by Isaac.
    
    Closes #181 from zhengruifeng/simplify-dataframe-reader.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/readwriter.rs | 142 +++++----------------------------
 1 file changed, 20 insertions(+), 122 deletions(-)

diff --git a/crates/spark-connect/src/readwriter.rs 
b/crates/spark-connect/src/readwriter.rs
index 5891641..9292288 100644
--- a/crates/spark-connect/src/readwriter.rs
+++ b/crates/spark-connect/src/readwriter.rs
@@ -79,13 +79,13 @@ impl DataFrameReader {
         let paths = path.map(|p| vec![p.to_string()]);
         let plan = LogicalPlan::Read {
             read_type: ReadType::DataSource {
-                format: self.format.clone(),
+                format: self.format,
                 schema: if self.schema.is_empty() {
                     None
                 } else {
-                    Some(self.schema.clone())
+                    Some(self.schema)
                 },
-                options: self.options.clone(),
+                options: self.options,
                 paths: paths.unwrap_or_default(),
                 predicates: vec![],
                 source_name: None,
@@ -99,7 +99,7 @@ impl DataFrameReader {
     pub fn changes(self, table_name: &str) -> DataFrame {
         let plan = LogicalPlan::RelationChanges {
             table_name: table_name.to_string(),
-            options: self.options.clone(),
+            options: self.options,
             is_streaming: None,
         };
         DataFrame::new(self.session, plan)
@@ -110,7 +110,7 @@ impl DataFrameReader {
         let plan = LogicalPlan::Read {
             read_type: ReadType::NamedTable {
                 table_name: table_name.to_string(),
-                options: self.options.clone(),
+                options: self.options,
             },
             is_streaming: false,
         };
@@ -118,135 +118,33 @@ impl DataFrameReader {
     }
 
     /// Read JSON data.
-    pub fn json(mut self, path: &str) -> DataFrame {
-        self.format = Some("json".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn json(self, path: &str) -> DataFrame {
+        self.format("json").load(Some(path))
     }
 
     /// Read Parquet data.
-    pub fn parquet(mut self, path: &str) -> DataFrame {
-        self.format = Some("parquet".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn parquet(self, path: &str) -> DataFrame {
+        self.format("parquet").load(Some(path))
     }
 
     /// Read CSV data.
-    pub fn csv(mut self, path: &str) -> DataFrame {
-        self.format = Some("csv".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn csv(self, path: &str) -> DataFrame {
+        self.format("csv").load(Some(path))
     }
 
     /// Read ORC data.
-    pub fn orc(mut self, path: &str) -> DataFrame {
-        self.format = Some("orc".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn orc(self, path: &str) -> DataFrame {
+        self.format("orc").load(Some(path))
     }
 
     /// Read text data.
-    pub fn text(mut self, path: &str) -> DataFrame {
-        self.format = Some("text".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn text(self, path: &str) -> DataFrame {
+        self.format("text").load(Some(path))
     }
 
     /// Read XML file(s). Mirrors `DataFrameReader.xml`.
-    pub fn xml(mut self, path: &str) -> DataFrame {
-        self.format = Some("xml".to_string());
-        let paths = vec![path.to_string()];
-        let plan = LogicalPlan::Read {
-            read_type: ReadType::DataSource {
-                format: self.format.clone(),
-                schema: if self.schema.is_empty() {
-                    None
-                } else {
-                    Some(self.schema.clone())
-                },
-                options: self.options.clone(),
-                paths,
-                predicates: vec![],
-                source_name: None,
-            },
-            is_streaming: false,
-        };
-        DataFrame::new(self.session, plan)
+    pub fn xml(self, path: &str) -> DataFrame {
+        self.format("xml").load(Some(path))
     }
 
     /// Read from JDBC data source.
@@ -258,13 +156,13 @@ impl DataFrameReader {
 
         let plan = LogicalPlan::Read {
             read_type: ReadType::DataSource {
-                format: self.format.clone(),
+                format: self.format,
                 schema: if self.schema.is_empty() {
                     None
                 } else {
-                    Some(self.schema.clone())
+                    Some(self.schema)
                 },
-                options: self.options.clone(),
+                options: self.options,
                 paths: vec![],
                 predicates: predicates.unwrap_or_default(),
                 source_name: None,


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to