This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 70688ec  [SPARK-61119][CONNECT] Set the element type of the array 
literals of the Rust core, as PySpark does
70688ec is described below

commit 70688ec07ff87ea5cd4d685247074feeaa2c2b7f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 23:38:19 2026 +0900

    [SPARK-61119][CONNECT] Set the element type of the array literals of the 
Rust core, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61119](https://issues.apache.org/jira/browse/SPARK-61119): The array 
literals of the Rust core carry the type of their elements, as PySpark's do.
    
    `LiteralExpression::Array::to_proto` set the elements of `Literal.Array`, 
but not its `element_type`. It now sets it, as PySpark's 
`LiteralExpression.to_plan` does, also for the arrays nested in an array. The 
field is deprecated in the proto in favor of the `data_type` of the literal, 
but it is what PySpark 4.3.0 writes and what the server reads.
    
    ### Why are the changes needed?
    
    A Spark 4.3.0 server rejects an array literal without an element type, also 
one of elements, with `CONNECT_INVALID_PLAN.ARRAY_LITERAL_MISSING_DATA_TYPE`. 
So every column of `Expression::literal(LiteralExpression::Array { .. })` of 
the Rust API failed on the server. The Python client builds arrays with the 
`array` function, so it was not affected.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, for users of the Rust API: array literals work on the server.
    
    ### How was this patch tested?
    
    - New Rust tests (`array_literal.rs`):
      - the element type of an empty array literal, and of a nested array and 
of its elements
      - against the server, a DataFrame of an array literal of elements and an 
empty one of strings, of the schema `struct<a:array<int>,e:array<string>>`
    
      Both fail without this change, the live one with 
`ARRAY_LITERAL_MISSING_DATA_TYPE`.
    - The unit test of the proto of an array literal checks its element type.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2165 passed, 8 skipped), and the 
live drop-in and Arrow suites (173 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #315 from HyukjinKwon/SPARK-61119-array-literal-type.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/expression.rs      | 12 +++-
 crates/spark-connect/tests/array_literal.rs | 96 +++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index 4c3b1ba..6016b52 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -577,10 +577,17 @@ impl LiteralExpression {
                 proto::expression::literal::LiteralType::Time(time)
             }
             LiteralExpression::Array {
-                element_type: _,
+                element_type,
                 elements,
             } => {
                 let mut array = proto::expression::literal::Array::default();
+                // The type of the elements, which the server requires of an 
array literal
+                // (`ARRAY_LITERAL_MISSING_DATA_TYPE`), as PySpark's 
`LiteralExpression.to_plan`
+                // writes it, also where the proto deprecates the field.
+                #[allow(deprecated)]
+                {
+                    array.element_type = Some(element_type.to_proto());
+                }
                 for elem in elements {
                     let elem_proto = elem.to_proto();
                     if let Some(proto::expression::ExprType::Literal(lit)) = 
elem_proto.expr_type {
@@ -2152,6 +2159,9 @@ mod tests {
             if let Some(proto::expression::literal::LiteralType::Array(arr)) = 
literal.literal_type
             {
                 assert_eq!(arr.elements.len(), 2);
+                #[allow(deprecated)]
+                let element_type = arr.element_type;
+                assert_eq!(element_type, Some(DataType::Integer.to_proto()));
             } else {
                 panic!("Expected array literal type");
             }
diff --git a/crates/spark-connect/tests/array_literal.rs 
b/crates/spark-connect/tests/array_literal.rs
new file mode 100644
index 0000000..e8cf09f
--- /dev/null
+++ b/crates/spark-connect/tests/array_literal.rs
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The array literals of the core, as PySpark's `LiteralExpression.to_plan` 
writes them: of the
+//! type of their elements, which the server requires. The tests of a server 
run when
+//! SPARK_REMOTE is set.
+
+use spark_connect::expression::{Expression, LiteralExpression};
+use spark_connect::types::DataType;
+use spark_connect::{Column, SparkSession};
+use spark_connect_proto as proto;
+
+fn string() -> DataType {
+    DataType::String {
+        collation: String::new(),
+    }
+}
+
+fn array(element_type: DataType, elements: Vec<LiteralExpression>) -> 
LiteralExpression {
+    LiteralExpression::Array {
+        element_type: Box::new(element_type),
+        elements,
+    }
+}
+
+#[allow(deprecated)]
+fn element_type(literal: &proto::expression::Literal) -> 
Option<proto::DataType> {
+    match &literal.literal_type {
+        Some(proto::expression::literal::LiteralType::Array(array)) => 
array.element_type.clone(),
+        _ => panic!("expected an array literal, got {literal:?}"),
+    }
+}
+
+#[test]
+fn array_literals_are_of_the_type_of_their_elements() {
+    let empty = array(string(), vec![]).to_literal_proto();
+    assert_eq!(element_type(&empty), Some(string().to_proto()));
+
+    let inner = DataType::Array {
+        element_type: Box::new(DataType::Long),
+        contains_null: true,
+    };
+    let nested = array(
+        inner.clone(),
+        vec![array(DataType::Long, vec![LiteralExpression::long(1)])],
+    )
+    .to_literal_proto();
+    assert_eq!(element_type(&nested), Some(inner.to_proto()));
+    let Some(proto::expression::literal::LiteralType::Array(outer)) = 
&nested.literal_type else {
+        unreachable!()
+    };
+    assert_eq!(
+        element_type(&outer.elements[0]),
+        Some(DataType::Long.to_proto())
+    );
+}
+
+#[test]
+fn array_literals_run_on_the_server() {
+    let Ok(remote) = std::env::var("SPARK_REMOTE") else {
+        return;
+    };
+    let session = SparkSession::builder()
+        .remote(&remote)
+        .get_or_create()
+        .unwrap();
+    let literal = |value: LiteralExpression| 
Column::new(Expression::literal(value));
+    let df = session.range(1).unwrap().select([
+        literal(array(
+            DataType::Integer,
+            vec![LiteralExpression::int(1), LiteralExpression::int(2)],
+        ))
+        .alias("a"),
+        literal(array(string(), vec![])).alias("e"),
+    ]);
+    assert_eq!(
+        df.schema().unwrap().simple_string(),
+        "struct<a:array<int>,e:array<string>>"
+    );
+    let rows = df.collect().unwrap();
+    assert_eq!(rows.len(), 1);
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to