mbutrovich commented on code in PR #331:
URL: https://github.com/apache/datafusion-spark/pull/331#discussion_r4605643879
##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{is_orderable, spark_sql_type};
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray,
OffsetSizeTrait,
+ PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+ DataType, Date32Type, Decimal128Type, Field, Float32Type, Float64Type,
Int8Type,
+ Int16Type, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType,
+ TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the maximum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map and INTERVAL
elements, at any \
+ depth, are an error, as in Spark. DataFusion's built-in of the same
name differs: it \
+ orders -0.0 before 0.0 rather than returning the first of them, sorts
NULLs nested \
+ inside arrays and structs after other values, rejects a NULL argument
at planning \
+ instead of returning NULL, and fails at execution on map elements.",
+ syntax_example = "array_max(array)",
+ sql_example = r#"```sql
+> SELECT array_max(array(1, 20, NULL, 3));
+20
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMax {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMax {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMax {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMax {
+ fn name(&self) -> &str {
+ "array_max"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Greater)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Spark-compatible `array_min` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_min>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the minimum element of the array, using Spark's
ordering. NULL \
+ elements are skipped, and an empty array or an array of only NULL
elements returns \
+ NULL. For floating point values, also inside nested arrays and
structs, NaN is \
+ greater than every other value, including infinity, and -0.0 is equal
to 0.0. On a \
+ tie the first of the tied elements is returned as it is, so the sign
of a zero and the \
+ bits of a NaN are kept. Strings and binary values compare bytewise.
Arrays compare \
+ element by element and then by length, and structs compare field by
field; NULLs \
+ nested inside them sort before other values. Map and INTERVAL
elements, at any \
+ depth, are an error, as in Spark. DataFusion's built-in of the same
name differs: it \
+ orders -0.0 before 0.0 rather than returning the first of them, sorts
NULLs nested \
+ inside arrays and structs after other values, rejects a NULL argument
at planning \
+ instead of returning NULL, and fails at execution on map elements.",
+ syntax_example = "array_min(array)",
+ sql_example = r#"```sql
+> SELECT array_min(array(1, 20, NULL, 3));
+1
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMin {
+ signature: Signature,
+}
+
+impl Default for SparkArrayMin {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayMin {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayMin {
+ fn name(&self) -> &str {
+ "array_min"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ element_type(self.name(), arg_types)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ coerce_array(self.name(), arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ invoke(self.name(), args, Ordering::Less)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// The element type of the array argument. A NULL-typed argument, which Spark
accepts,
+/// returns the NULL type.
+fn element_type(name: &str, arg_types: &[DataType]) -> Result<DataType> {
+ let [array] = take_function_args(name, arg_types)?;
+ match array {
+ DataType::Null => Ok(DataType::Null),
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
+ other => unexpected_input_type(name, other),
+ }
+}
+
+/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept:
an array of an
+/// orderable type, or NULL. Fixed-size lists are cast to lists.
+fn coerce_array(name: &str, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array] = take_function_args(name, arg_types)?;
+ let coerced = match array {
+ DataType::Null | DataType::List(_) | DataType::LargeList(_) =>
array.clone(),
+ DataType::FixedSizeList(field, _) =>
DataType::List(Arc::new(Field::new(
+ field.name(),
+ field.data_type().clone(),
+ true,
+ ))),
+ other => return unexpected_input_type(name, other),
+ };
Review Comment:
A list of dictionary-encoded doubles keeps its `Dictionary` element type
here, so it goes to `generic_extrema`. `normalize_array` doesn't look inside
dictionaries, and the comparator then orders `-0.0` below `0.0`:
```
SELECT CAST(array_max(c) AS VARCHAR), CAST(array_min(d) AS VARCHAR),
CAST(array_max(e) AS VARCHAR), CAST(array_min(f) AS VARCHAR)
FROM (SELECT arrow_cast(array('-0.0'::double, 0.0::double),
'List(Dictionary(Int32, Float64))') AS c,
arrow_cast(array(0.0::double, '-0.0'::double),
'List(Dictionary(Int32, Float64))') AS d,
array('-0.0'::double, 0.0::double) AS e,
array(0.0::double, '-0.0'::double) AS f)
-- 0.0 -0.0 -0.0 0.0
```
The plain lists `e` and `f` return the first tied element, as Spark does,
and the dictionary lists `c` and `d` don't. Spark has no dictionary types, but
engines that build on these functions pass dictionary-encoded columns, for
example Sail's
[`rtrim`](https://github.com/lakehq/sail/blob/93e11f8119b8ad36be6193c019c3baf546febaf8/crates/sail-function/src/scalar/string/rtrim.rs#L31-L34).
Could `coerce_array` replace a dictionary element type with its value type,
the way #345's `tightest_common_type` treats a dictionary argument, with a
float case in `array_extrema.slt`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]