This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 70688ec [SPARK-61119][CONNECT] Set the element type of the array
literals of the Rust core, as PySpark does
70688ec is described below
commit 70688ec07ff87ea5cd4d685247074feeaa2c2b7f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 23:38:19 2026 +0900
[SPARK-61119][CONNECT] Set the element type of the array literals of the
Rust core, as PySpark does
### What changes were proposed in this pull request?
[SPARK-61119](https://issues.apache.org/jira/browse/SPARK-61119): The array
literals of the Rust core carry the type of their elements, as PySpark's do.
`LiteralExpression::Array::to_proto` set the elements of `Literal.Array`,
but not its `element_type`. It now sets it, as PySpark's
`LiteralExpression.to_plan` does, also for the arrays nested in an array. The
field is deprecated in the proto in favor of the `data_type` of the literal,
but it is what PySpark 4.3.0 writes and what the server reads.
### Why are the changes needed?
A Spark 4.3.0 server rejects an array literal without an element type, also
one of elements, with `CONNECT_INVALID_PLAN.ARRAY_LITERAL_MISSING_DATA_TYPE`.
So every column of `Expression::literal(LiteralExpression::Array { .. })` of
the Rust API failed on the server. The Python client builds arrays with the
`array` function, so it was not affected.
### Does this PR introduce _any_ user-facing change?
Yes, for users of the Rust API: array literals work on the server.
### How was this patch tested?
- New Rust tests (`array_literal.rs`):
- the element type of an empty array literal, and of a nested array and
of its elements
- against the server, a DataFrame of an array literal of elements and an
empty one of strings, of the schema `struct<a:array<int>,e:array<string>>`
Both fail without this change, the live one with
`ARRAY_LITERAL_MISSING_DATA_TYPE`.
- The unit test of the proto of an array literal checks its element type.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2165 passed, 8 skipped), and the
live drop-in and Arrow suites (173 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #315 from HyukjinKwon/SPARK-61119-array-literal-type.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/expression.rs | 12 +++-
crates/spark-connect/tests/array_literal.rs | 96 +++++++++++++++++++++++++++++
2 files changed, 107 insertions(+), 1 deletion(-)
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index 4c3b1ba..6016b52 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -577,10 +577,17 @@ impl LiteralExpression {
proto::expression::literal::LiteralType::Time(time)
}
LiteralExpression::Array {
- element_type: _,
+ element_type,
elements,
} => {
let mut array = proto::expression::literal::Array::default();
+ // The type of the elements, which the server requires of an
array literal
+ // (`ARRAY_LITERAL_MISSING_DATA_TYPE`), as PySpark's
`LiteralExpression.to_plan`
+ // writes it, also where the proto deprecates the field.
+ #[allow(deprecated)]
+ {
+ array.element_type = Some(element_type.to_proto());
+ }
for elem in elements {
let elem_proto = elem.to_proto();
if let Some(proto::expression::ExprType::Literal(lit)) =
elem_proto.expr_type {
@@ -2152,6 +2159,9 @@ mod tests {
if let Some(proto::expression::literal::LiteralType::Array(arr)) =
literal.literal_type
{
assert_eq!(arr.elements.len(), 2);
+ #[allow(deprecated)]
+ let element_type = arr.element_type;
+ assert_eq!(element_type, Some(DataType::Integer.to_proto()));
} else {
panic!("Expected array literal type");
}
diff --git a/crates/spark-connect/tests/array_literal.rs
b/crates/spark-connect/tests/array_literal.rs
new file mode 100644
index 0000000..e8cf09f
--- /dev/null
+++ b/crates/spark-connect/tests/array_literal.rs
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The array literals of the core, as PySpark's `LiteralExpression.to_plan`
writes them: of the
+//! type of their elements, which the server requires. The tests of a server
run when
+//! SPARK_REMOTE is set.
+
+use spark_connect::expression::{Expression, LiteralExpression};
+use spark_connect::types::DataType;
+use spark_connect::{Column, SparkSession};
+use spark_connect_proto as proto;
+
+fn string() -> DataType {
+ DataType::String {
+ collation: String::new(),
+ }
+}
+
+fn array(element_type: DataType, elements: Vec<LiteralExpression>) ->
LiteralExpression {
+ LiteralExpression::Array {
+ element_type: Box::new(element_type),
+ elements,
+ }
+}
+
+#[allow(deprecated)]
+fn element_type(literal: &proto::expression::Literal) ->
Option<proto::DataType> {
+ match &literal.literal_type {
+ Some(proto::expression::literal::LiteralType::Array(array)) =>
array.element_type.clone(),
+ _ => panic!("expected an array literal, got {literal:?}"),
+ }
+}
+
+#[test]
+fn array_literals_are_of_the_type_of_their_elements() {
+ let empty = array(string(), vec![]).to_literal_proto();
+ assert_eq!(element_type(&empty), Some(string().to_proto()));
+
+ let inner = DataType::Array {
+ element_type: Box::new(DataType::Long),
+ contains_null: true,
+ };
+ let nested = array(
+ inner.clone(),
+ vec![array(DataType::Long, vec![LiteralExpression::long(1)])],
+ )
+ .to_literal_proto();
+ assert_eq!(element_type(&nested), Some(inner.to_proto()));
+ let Some(proto::expression::literal::LiteralType::Array(outer)) =
&nested.literal_type else {
+ unreachable!()
+ };
+ assert_eq!(
+ element_type(&outer.elements[0]),
+ Some(DataType::Long.to_proto())
+ );
+}
+
+#[test]
+fn array_literals_run_on_the_server() {
+ let Ok(remote) = std::env::var("SPARK_REMOTE") else {
+ return;
+ };
+ let session = SparkSession::builder()
+ .remote(&remote)
+ .get_or_create()
+ .unwrap();
+ let literal = |value: LiteralExpression|
Column::new(Expression::literal(value));
+ let df = session.range(1).unwrap().select([
+ literal(array(
+ DataType::Integer,
+ vec![LiteralExpression::int(1), LiteralExpression::int(2)],
+ ))
+ .alias("a"),
+ literal(array(string(), vec![])).alias("e"),
+ ]);
+ assert_eq!(
+ df.schema().unwrap().simple_string(),
+ "struct<a:array<int>,e:array<string>>"
+ );
+ let rows = df.collect().unwrap();
+ assert_eq!(rows.len(), 1);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]