mbutrovich commented on code in PR #331:
URL: https://github.com/apache/datafusion-spark/pull/331#discussion_r4602612257
##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use super::array_position::contains_map;
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray,
OffsetSizeTrait,
+ PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+ DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field,
Float16Type,
+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the maximum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_max(array)",
+ sql_example = r#"```sql
+> SELECT array_max(array(1, 20, NULL, 3));
+20
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMax {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMax {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMax {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMax {
+ fn name(&self) -> &str {
+ "array_max"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Greater)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Spark-compatible `array_min` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_min>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the minimum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_min(array)",
+ sql_example = r#"```sql
+> SELECT array_min(array(1, 20, NULL, 3));
+1
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMin {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMin {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMin {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMin {
+ fn name(&self) -> &str {
+ "array_min"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Less)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// The element type of the array argument. A NULL-typed argument, which Spark
accepts,
+/// returns the NULL type.
+fn element_type(name: &str, arg_types: &[DataType]) -> Result<DataType> {
+ let [array] = take_function_args(name, arg_types)?;
+ match array {
+ DataType::Null => Ok(DataType::Null),
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
+ other => unexpected_input_type(name, other),
+ }
+}
+
+/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept:
an array of an
+/// orderable type, or NULL. Fixed-size lists are cast to lists.
+fn coerce_array(name: &str, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array] = take_function_args(name, arg_types)?;
+ let coerced = match array {
+ DataType::Null | DataType::List(_) | DataType::LargeList(_) =>
array.clone(),
+ DataType::FixedSizeList(field, _) =>
DataType::List(Arc::new(Field::new(
+ field.name(),
+ field.data_type().clone(),
+ true,
+ ))),
+ other => return unexpected_input_type(name, other),
+ };
+ let element = element_type(name, std::slice::from_ref(&coerced))?;
+ if contains_map(&element) {
+ return plan_err!(
+ "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not
support \
+ ordering on type {element}"
+ );
+ }
Review Comment:
Spark can't order `CalendarIntervalType`. This crate's `make_interval`
returns it as `Interval(MonthDayNano)`, and Comet maps it to the same Arrow
type
([`serde.rs`](https://github.com/apache/datafusion-comet/blob/ba16e1a53127c90ba00fdc7058703945a8bb9f34/native/core/src/execution/serde.rs#L106-L108)).
[`OrderUtils.isOrderable`](https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/sql/api/src/main/scala/org/apache/spark/sql/catalyst/expressions/OrderUtils.scala#L26-L38)
accepts only atomic types and arrays and structs of them, and
[`ArrayMax`](https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala#L2428-L2435)
checks it. For `SELECT array_max(array(make_interval(1)))`, Spark 4.2.0 raises:
```
[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] ... The `array_max` does not
support ordering on type "INTERVAL".
```
Here `array_max(array(make_interval(0, 0, 0, 40), make_interval(0, 1)))`
returns `1 mons`, because Arrow orders by months, then days, then nanoseconds.
Could the check reject `Interval(MonthDayNano)` at any depth, not only maps,
with an slt error case? #332's `is_orderable` needs the same fix. My review of
#332 proposes one shared module for the orderability check, the type coercion,
and the type names, so the check could live there.
##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use super::array_position::contains_map;
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray,
OffsetSizeTrait,
+ PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+ DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field,
Float16Type,
+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the maximum element of the array, using Spark's
ordering. NULL \
Review Comment:
DataFusion has its own `array_max` and `array_min`
([`min_max.rs`](https://github.com/apache/datafusion/blob/cce6374e6256b3b8892ec60674578cf32bc3a32c/datafusion/functions-nested/src/min_max.rs#L22)),
and
[`with_spark_features`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/crates/spark/src/session_state.rs#L29)
replaces them with these.
[`CONTRIBUTING.md`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/CONTRIBUTING.md?plain=1#L47-L48)
asks the description to say how a function differs from the built-in of the
same name. Could both descriptions (here and at line 101) say that, as #332
does for `greatest` and `least`, with a test for each difference?
##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use super::array_position::contains_map;
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray,
OffsetSizeTrait,
+ PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+ DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field,
Float16Type,
+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the maximum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_max(array)",
+ sql_example = r#"```sql
+> SELECT array_max(array(1, 20, NULL, 3));
+20
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMax {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMax {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMax {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMax {
+ fn name(&self) -> &str {
+ "array_max"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Greater)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Spark-compatible `array_min` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_min>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the minimum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_min(array)",
+ sql_example = r#"```sql
+> SELECT array_min(array(1, 20, NULL, 3));
+1
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMin {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMin {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMin {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMin {
+ fn name(&self) -> &str {
+ "array_min"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Less)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// The element type of the array argument. A NULL-typed argument, which Spark
accepts,
+/// returns the NULL type.
+fn element_type(name: &str, arg_types: &[DataType]) -> Result<DataType> {
+ let [array] = take_function_args(name, arg_types)?;
+ match array {
+ DataType::Null => Ok(DataType::Null),
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
+ other => unexpected_input_type(name, other),
+ }
+}
+
+/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept:
an array of an
+/// orderable type, or NULL. Fixed-size lists are cast to lists.
+fn coerce_array(name: &str, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array] = take_function_args(name, arg_types)?;
+ let coerced = match array {
+ DataType::Null | DataType::List(_) | DataType::LargeList(_) =>
array.clone(),
+ DataType::FixedSizeList(field, _) =>
DataType::List(Arc::new(Field::new(
+ field.name(),
+ field.data_type().clone(),
+ true,
+ ))),
+ other => return unexpected_input_type(name, other),
+ };
+ let element = element_type(name, std::slice::from_ref(&coerced))?;
+ if contains_map(&element) {
+ return plan_err!(
+ "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not
support \
+ ordering on type {element}"
+ );
+ }
+ Ok(vec![coerced])
+}
+
+fn unexpected_input_type<T>(name: &str, data_type: &DataType) -> Result<T> {
+ plan_err!(
+ "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of
`{name}` \
+ requires the \"ARRAY\" type, however it has the type {data_type}"
+ )
+}
+
+fn invoke(
+ name: &str,
+ args: ScalarFunctionArgs,
+ target: Ordering,
+) -> Result<ColumnarValue> {
+ let [arg] = take_function_args(name, &args.args)?;
+ let element_type = args.return_field.data_type();
+ let array = match arg {
+ ColumnarValue::Array(array) => Arc::clone(array),
+ ColumnarValue::Scalar(scalar) => scalar.to_array()?,
+ };
+
+ let result = match array.data_type() {
+ DataType::Null => new_null_array(element_type, array.len()),
+ // All the elements are NULL, so every result is NULL.
+ _ if element_type.is_null() => new_null_array(element_type,
array.len()),
+ DataType::List(_) => extrema(array.as_list::<i32>(), target)?,
+ DataType::LargeList(_) => extrema(array.as_list::<i64>(), target)?,
+ other => return exec_err!("{name} does not support type {other}"),
+ };
+
+ if matches!(arg, ColumnarValue::Scalar(_)) {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+}
+
+/// Returns, for each list, its first non-NULL element that no other element
orders
+/// `target` of, or NULL when the list is NULL, empty, or holds only NULL
elements.
+fn extrema<O: OffsetSizeTrait>(
+ list: &GenericListArray<O>,
+ target: Ordering,
+) -> Result<ArrayRef> {
+ let offsets = list.offsets();
+ // Only the values that this (possibly sliced) list refers to are compared.
+ let base = offsets[0].as_usize();
+ let len = offsets[offsets.len() - 1].as_usize() - base;
+ let values = list.values().slice(base, len);
+
+ // Primitive elements take a typed path: no copy of the values, no
type-erased comparator
+ // and no `take`.
+ macro_rules! primitive {
+ ($t:ty, $cmp:expr) => {
+ return Ok(Arc::new(primitive_extrema::<$t, O>(
+ list,
+ values.as_primitive::<$t>(),
+ base,
+ target,
+ $cmp,
+ )))
+ };
+ }
+ macro_rules! ordered {
+ ($t:ty) => {
+ primitive!($t, |a, b| a.cmp(&b))
+ };
+ }
+ match values.data_type() {
+ DataType::Int8 => ordered!(Int8Type),
+ DataType::Int16 => ordered!(Int16Type),
+ DataType::Int32 => ordered!(Int32Type),
+ DataType::Int64 => ordered!(Int64Type),
+ DataType::UInt8 => ordered!(UInt8Type),
+ DataType::UInt16 => ordered!(UInt16Type),
+ DataType::UInt32 => ordered!(UInt32Type),
+ DataType::UInt64 => ordered!(UInt64Type),
+ DataType::Decimal128(_, _) => ordered!(Decimal128Type),
+ DataType::Decimal256(_, _) => ordered!(Decimal256Type),
+ DataType::Date32 => ordered!(Date32Type),
+ DataType::Date64 => ordered!(Date64Type),
+ DataType::Timestamp(TimeUnit::Second, _) =>
ordered!(TimestampSecondType),
+ DataType::Timestamp(TimeUnit::Millisecond, _) => {
+ ordered!(TimestampMillisecondType)
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ ordered!(TimestampMicrosecondType)
+ }
+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+ ordered!(TimestampNanosecondType)
+ }
+ DataType::Float16 => primitive!(Float16Type, spark_float_cmp),
+ DataType::Float32 => primitive!(Float32Type, spark_float_cmp),
+ DataType::Float64 => primitive!(Float64Type, spark_float_cmp),
Review Comment:
Spark never produces `UInt8` through `UInt64`, `Float16`, `Date64`, or
`Decimal256` (see "Supported types" in
[CONTRIBUTING.md](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/CONTRIBUTING.md?plain=1#L31-L34)),
and no test reaches these arms. `generic_extrema` already handles them. Could
we drop them, so that every typed arm is one the tests check against the
generic path?
##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use super::array_position::contains_map;
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray,
OffsetSizeTrait,
+ PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+ DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field,
Float16Type,
+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the maximum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_max(array)",
+ sql_example = r#"```sql
+> SELECT array_max(array(1, 20, NULL, 3));
+20
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMax {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMax {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMax {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMax {
+ fn name(&self) -> &str {
+ "array_max"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Greater)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Spark-compatible `array_min` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_min>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the minimum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map elements, at any
depth, are an \
+ error, as in Spark.",
+ syntax_example = "array_min(array)",
+ sql_example = r#"```sql
+> SELECT array_min(array(1, 20, NULL, 3));
+1
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMin {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMin {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMin {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMin {
+ fn name(&self) -> &str {
+ "array_min"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Less)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// The element type of the array argument. A NULL-typed argument, which Spark
accepts,
+/// returns the NULL type.
+fn element_type(name: &str, arg_types: &[DataType]) -> Result<DataType> {
+ let [array] = take_function_args(name, arg_types)?;
+ match array {
+ DataType::Null => Ok(DataType::Null),
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
+ other => unexpected_input_type(name, other),
+ }
+}
+
+/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept:
an array of an
+/// orderable type, or NULL. Fixed-size lists are cast to lists.
+fn coerce_array(name: &str, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array] = take_function_args(name, arg_types)?;
+ let coerced = match array {
+ DataType::Null | DataType::List(_) | DataType::LargeList(_) =>
array.clone(),
+ DataType::FixedSizeList(field, _) =>
DataType::List(Arc::new(Field::new(
+ field.name(),
+ field.data_type().clone(),
+ true,
+ ))),
+ other => return unexpected_input_type(name, other),
+ };
+ let element = element_type(name, std::slice::from_ref(&coerced))?;
+ if contains_map(&element) {
+ return plan_err!(
+ "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not
support \
+ ordering on type {element}"
+ );
+ }
+ Ok(vec![coerced])
+}
+
+fn unexpected_input_type<T>(name: &str, data_type: &DataType) -> Result<T> {
+ plan_err!(
+ "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of
`{name}` \
+ requires the \"ARRAY\" type, however it has the type {data_type}"
+ )
Review Comment:
These messages print the Arrow type where Spark prints its SQL type name.
`SELECT array_max(1)` gives `... however it has the type Int64`, and
`array_max(array(map(1, 2)))` gives `... on type Map("entries": non-null
Struct("key": non-null Int64, "value": Int64), unsorted)`. Spark 4.2.0 gives
`... however "1" has the type "INT". SQLSTATE: 42K09` and `... on type
"MAP<INT, INT>". SQLSTATE: 42K09`. Could we render the type the way Spark does
and end with the period and SQLSTATE, as in the
[templates](https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/common/utils/src/main/resources/error/error-conditions.json#L1751-L1754)?
The hand-built messages on `main` include the SQLSTATE, for example
[`encode.rs`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/crates/spark/src/function/string/encode.rs#L118).
#332 adds `spark_sql_type` to render Spark's type names, and my review of #332
proposes moving it into a shared module
that this function could use.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]